diff --git a/CMakeLists.txt b/CMakeLists.txt index 4f1ab11c3..59b02d9bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ endif() # This is the top level CMake file for the SCOREC build cmake_minimum_required(VERSION 3.8) -project(SCOREC VERSION 2.2.8 LANGUAGES CXX C) +project(SCOREC VERSION 2.3.0 LANGUAGES CXX C) include(cmake/bob.cmake) include(cmake/xsdk.cmake) diff --git a/apf/apf.h b/apf/apf.h index 7db3f37a2..9a9ba849a 100644 --- a/apf/apf.h +++ b/apf/apf.h @@ -12,6 +12,10 @@ #include "apfNew.h" #include "apfDynamicArray.h" +namespace pcu{ + class PCU; +} + #include #include #include @@ -460,7 +464,7 @@ class Integrator * process-local integrations into a global mesh integration, * if that is the user's goal. */ - virtual void parallelReduce(); + virtual void parallelReduce(pcu::PCU*); protected: int order; int ipnode; diff --git a/apf/apfCGNS.cc b/apf/apfCGNS.cc index 4debbb58b..4bd3499b0 100644 --- a/apf/apfCGNS.cc +++ b/apf/apfCGNS.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include "apfMesh.h" #include "apfNumbering.h" #include "apfNumberingClass.h" @@ -13,7 +12,6 @@ #include "apfFieldData.h" #include #include -// #include #include #include @@ -47,7 +45,7 @@ static Count count(apf::Mesh *m, int dim) { const int local = apf::countOwned(m, dim); int total = local; - PCU_Add_Ints(&total, 1); // size of total array + m->getPCU()->Add(&total, 1); // size of total array return std::make_pair(total, local); } @@ -290,7 +288,7 @@ void WriteFields(const CGNS &cgns, const std::vectorgetPCU()->Add(&size, 1); // size of total array // oddness of the api rmin[1] = rmin[0]; @@ -505,7 +503,7 @@ CellElementReturn WriteElements(const CGNS &cgns, apf::Mesh *m, apf::GlobalNumbe m->end(cellIter); numbersByElementType[o] = counter; int total = counter; - PCU_Add_Ints(&total, 1); // size of total array + m->getPCU()->Add(&total, 1); // size of total array globalNumbersByElementType[o] = total; } cgsize_t allTotal = std::accumulate(globalNumbersByElementType.begin(), globalNumbersByElementType.end(), 0); @@ -546,12 +544,12 @@ CellElementReturn WriteElements(const CGNS &cgns, apf::Mesh *m, apf::GlobalNumbe if (cgp_section_write(cgns.index, cgns.base, cgns.zone, name.c_str(), cgnsElementOrder[o], globalStart, globalEnd, 0, §ionNumber)) // global start, end within the file for that element type cgp_error_exit(); - std::vector allNumbersForThisType(PCU_Comm_Peers(), 0); + std::vector allNumbersForThisType(m->getPCU()->Peers(), 0); MPI_Allgather(&numbersByElementType[o], 1, MPI_INT, allNumbersForThisType.data(), 1, - MPI_INT, PCU_Get_Comm()); + MPI_INT, m->getPCU()->GetMPIComm()); cgsize_t num = 0; - for (int i = 0; i < PCU_Comm_Self(); i++) + for (int i = 0; i < m->getPCU()->Self(); i++) num += allNumbersForThisType[i]; cgsize_t elStart = globalStart + num; @@ -635,7 +633,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, int startOfBCBlock = startingLocation + 1; const int number = bc.second.size(); int total = number; - PCU_Add_Ints(&total, 1); // size of total array + m->getPCU()->Add(&total, 1); // size of total array if (total > 0) { const auto allEnd = startOfBCBlock + total - 1; //one-based @@ -657,12 +655,12 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, } } - std::vector allNumbersForThisType(PCU_Comm_Peers(), 0); + std::vector allNumbersForThisType(m->getPCU()->Peers(), 0); MPI_Allgather(&number, 1, MPI_INT, allNumbersForThisType.data(), 1, - MPI_INT, PCU_Get_Comm()); + MPI_INT, m->getPCU()->GetMPIComm()); cgsize_t num = 0; - for (int i = 0; i < PCU_Comm_Self(); i++) + for (int i = 0; i < m->getPCU()->Self(); i++) num += allNumbersForThisType[i]; cgsize_t elStart = startOfBCBlock + num; @@ -685,26 +683,26 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, cacheEnd = allEnd; } } - std::vector cacheStarts(PCU_Comm_Peers(), 0); + std::vector cacheStarts(m->getPCU()->Peers(), 0); MPI_Allgather(&cacheStart, 1, MPI_INT, cacheStarts.data(), 1, - MPI_INT, PCU_Get_Comm()); - std::vector cacheEnds(PCU_Comm_Peers(), 0); + MPI_INT, m->getPCU()->GetMPIComm()); + std::vector cacheEnds(m->getPCU()->Peers(), 0); MPI_Allgather(&cacheEnd, 1, MPI_INT, cacheEnds.data(), 1, - MPI_INT, PCU_Get_Comm()); + MPI_INT, m->getPCU()->GetMPIComm()); return std::make_pair(cacheStarts, cacheEnds); }; - const auto globalElementList = [](const std::vector &bcList, std::vector &allElements) { - std::vector sizes(PCU_Comm_Peers(), 0); // important initialiser + const auto globalElementList = [&m](const std::vector &bcList, std::vector &allElements) { + std::vector sizes(m->getPCU()->Peers(), 0); // important initialiser const int l = bcList.size(); MPI_Allgather(&l, 1, MPI_INT, sizes.data(), 1, - MPI_INT, PCU_Get_Comm()); + MPI_INT, m->getPCU()->GetMPIComm()); int totalLength = 0; for (const auto &i : sizes) totalLength += i; - std::vector displacement(PCU_Comm_Peers(), -1); + std::vector displacement(m->getPCU()->Peers(), -1); displacement[0] = 0; for (std::size_t i = 1; i < displacement.size(); i++) displacement[i] = displacement[i - 1] + sizes[i - 1]; @@ -712,7 +710,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, allElements.resize(totalLength); MPI_Allgatherv(bcList.data(), bcList.size(), MPI_INT, allElements.data(), sizes.data(), displacement.data(), MPI_INT, - PCU_Get_Comm()); + m->getPCU()->GetMPIComm()); }; const auto doVertexBC = [&](const auto &iter) { @@ -755,7 +753,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, for (const auto &p : iter->second) { const auto se = BCEntityAdder(EdgeLoop, p, startingLocation); - for (int i = 0; i < PCU_Comm_Peers(); i++) + for (int i = 0; i < m->getPCU()->Peers(); i++) { PCU_ALWAYS_ASSERT_VERBOSE(se.first[i] == se.first[0], "Must all be the same "); PCU_ALWAYS_ASSERT_VERBOSE(se.second[i] == se.second[0], "Must all be the same "); @@ -779,7 +777,7 @@ void AddBocosToMainBase(const CGNS &cgns, const CellElementReturn &cellResults, { const auto se = BCEntityAdder(FaceLoop, p, startingLocation); - for (int i = 0; i < PCU_Comm_Peers(); i++) + for (int i = 0; i < m->getPCU()->Peers(); i++) { PCU_ALWAYS_ASSERT_VERBOSE(se.first[i] == se.first[0], "Must all be the same "); PCU_ALWAYS_ASSERT_VERBOSE(se.second[i] == se.second[0], "Must all be the same "); @@ -1011,7 +1009,7 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap { static_assert(std::is_same::value, "cgsize_t not compiled as int"); - const auto myRank = PCU_Comm_Self(); + const auto myRank = m->getPCU()->Self(); const Count vertexCount = count(m, 0); const Count edgeCount = count(m, 1); const Count faceCount = count(m, 2); @@ -1031,7 +1029,7 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap // PCU_Barrier(); // } - PCU_Barrier(); + m->getPCU()->Barrier(); if (myRank == 0) { std::cout << "*******Global Mesh Stats*****************\n"; @@ -1048,14 +1046,14 @@ void WriteCGNS(const char *prefix, apf::Mesh *m, const apf::CGNSBCMap &cgnsBCMap sizes[2] = 0; // nodes are unsorted, as defined by api // Copy communicator - auto communicator = PCU_Get_Comm(); + auto communicator = m->getPCU()->GetMPIComm(); cgp_mpi_comm(communicator); // - cgp_pio_mode(CGNS_ENUMV(CGP_INDEPENDENT)); + cgp_pio_mode(CGP_INDEPENDENT); CGNS cgns; cgns.fname = std::string(prefix); - if (cgp_open(prefix, CGNS_ENUMV(CG_MODE_WRITE), &cgns.index)) + if (cgp_open(prefix, CG_MODE_WRITE, &cgns.index)) cgp_error_exit(); { diff --git a/apf/apfCavityOp.cc b/apf/apfCavityOp.cc index 39b91b8d1..568fd2287 100644 --- a/apf/apfCavityOp.cc +++ b/apf/apfCavityOp.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include "apfCavityOp.h" #include "apf.h" #include "apfMesh2.h" @@ -134,10 +133,10 @@ bool CavityOp::requestLocality(MeshEntity** entities, int count) bool CavityOp::sendPullRequests(std::vector& received) { - int done = PCU_Min_Int(requests.empty()); + int done = mesh->getPCU()->Min(requests.empty()); if (done) return false; /* throw in the local pull requests */ - int self = PCU_Comm_Self(); + int self = mesh->getPCU()->Self(); received.reserve(requests.size()); APF_ITERATE(Requests,requests,it) { @@ -147,7 +146,7 @@ bool CavityOp::sendPullRequests(std::vector& received) received.push_back(request); } /* now communicate the rest */ - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); APF_ITERATE(Requests,requests,it) { CopyArray remotes; @@ -156,18 +155,17 @@ bool CavityOp::sendPullRequests(std::vector& received) { int remotePart = rit->peer; MeshEntity* remoteEntity = rit->entity; - PCU_COMM_PACK(remotePart,remoteEntity); - } + mesh->getPCU()->Pack(remotePart,remoteEntity); } } requests.clear(); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) + mesh->getPCU()->Send(); + while (mesh->getPCU()->Listen()) { PullRequest request; - request.to = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) + request.to = mesh->getPCU()->Sender(); + while ( ! mesh->getPCU()->Unpacked()) { - PCU_COMM_UNPACK(request.e); + mesh->getPCU()->Unpack(request.e); received.push_back(request); } } diff --git a/apf/apfConstruct.cc b/apf/apfConstruct.cc index 40fdbbeb0..c72830f25 100644 --- a/apf/apfConstruct.cc +++ b/apf/apfConstruct.cc @@ -1,4 +1,3 @@ -#include #include "pcu_util.h" #include "apfConvert.h" #include "apfMesh2.h" @@ -15,7 +14,7 @@ static void constructVerts( { ModelEntity* interior = m->findModelEntity(m->getDimension(), 0); int end = nelem * apf::Mesh::adjacentCount[etype][0]; - int self2 = PCU_Comm_Self(); + int self2 = m->getPCU()->Self(); for (int i = 0; i < end; ++i) if ( ! result.count(conn[i])) { result[conn[i]] = m->createVert_(interior); @@ -43,12 +42,12 @@ static NewElements constructElements( return newElements; } -static Gid getMax(const GlobalToVert& globalToVert) +static Gid getMax(const GlobalToVert& globalToVert, pcu::PCU *PCUObj) { Gid max = -1; APF_CONST_ITERATE(GlobalToVert, globalToVert, it) max = std::max(max, it->first); - return PCU_Max_Long(max); // this is type-dependent + return PCUObj->Max(max); // this is type-dependent } @@ -59,7 +58,7 @@ static Gid getMax(const GlobalToVert& globalToVert) static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) { Gid ifirst=0; - int self2 = PCU_Comm_Self(); + int self2 = m->getPCU()->Self(); APF_ITERATE(GlobalToVert, globalToVert, it) { Gid gid = it->first; if(ifirst==0 || ifirst==13437400 ) { @@ -67,7 +66,7 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) } ifirst++; } - Gid max = getMax(globalToVert); // seems like we read this and know it already on every rank so why compute with global comm? + Gid max = getMax(globalToVert, m->getPCU()); // seems like we read this and know it already on every rank so why compute with global comm? ifirst=0; APF_ITERATE(GlobalToVert, globalToVert, it) { Gid gid = it->first; @@ -80,13 +79,13 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) ifirst++; } Gid total = max + 1; - int peers = PCU_Comm_Peers(); + int peers = m->getPCU()->Peers(); Gid quotientL = total / peers; int quotient = quotientL; Gid remainderL = total % peers; int remainder = remainderL; int mySize = quotient; - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); if (self == (peers - 1)) mySize += remainder; if (self == (peers - 1)) lion_eprint(1, "CR1 mysize=%d \n",mySize); @@ -94,7 +93,7 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) TmpParts tmpParts(mySize); /* if we have a vertex, send its global id to the broker for that global id */ - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(GlobalToVert, globalToVert, it) { Gid gid = it->first; if(gid < 0 || gid > max ){ @@ -102,49 +101,49 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) } Gid tmpL=gid / quotient; int tmpI=tmpL; int to = std::min(peers - 1,tmpI); - PCU_COMM_PACK(to, gid); + m->getPCU()->Pack(to, gid); } - PCU_Comm_Send(); + m->getPCU()->Send(); Gid myOffset = (long)self * quotient; if (self == (peers - 1)) lion_eprint(1, "CR5: self=%d,myOffset=%ld,quotient=%d \n",self,myOffset,quotient); /* brokers store all the part ids that sent messages for each global id */ - while (PCU_Comm_Receive()) { + while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); - int from = PCU_Comm_Sender(); + m->getPCU()->Unpack(gid); + int from = m->getPCU()->Sender(); Gid tmpL=gid - myOffset; // forcing 64 bit difference until we know it is safe int tmpI=tmpL; tmpParts.at(tmpI).push_back(from); } /* for each global id, send all associated part ids to all associated parts */ - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int i = 0; i < mySize; ++i) { std::vector& parts = tmpParts[i]; for (size_t j = 0; j < parts.size(); ++j) { int to = parts[j]; Gid gid = i + myOffset; int nparts = parts.size(); - PCU_COMM_PACK(to, gid); - PCU_COMM_PACK(to, nparts); + m->getPCU()->Pack(to, gid); + m->getPCU()->Pack(to, nparts); for (size_t k = 0; k < parts.size(); ++k) - PCU_COMM_PACK(to, parts[k]); + m->getPCU()->Pack(to, parts[k]); } } - PCU_Comm_Send(); + m->getPCU()->Send(); /* receiving a global id and associated parts, lookup the vertex and classify it on the partition model entity for that set of parts */ - while (PCU_Comm_Receive()) { + while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); int nparts; - PCU_COMM_UNPACK(nparts); + m->getPCU()->Unpack(nparts); Parts residence; for (int i = 0; i < nparts; ++i) { int part; - PCU_COMM_UNPACK(part); + m->getPCU()->Unpack(part); residence.insert(part); } MeshEntity* vert = globalToVert[gid]; @@ -157,8 +156,8 @@ static void constructResidence(Mesh2* m, GlobalToVert& globalToVert) pairs with parts in the residence of the vertex */ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert) { - int self = PCU_Comm_Self(); - PCU_Comm_Begin(); + int self = m->getPCU()->Self(); + m->getPCU()->Begin(); APF_ITERATE(GlobalToVert, globalToVert, it) { Gid gid = it->first; if(gid < 0 ) @@ -168,22 +167,22 @@ static void constructRemotes(Mesh2* m, GlobalToVert& globalToVert) m->getResidence(vert, residence); APF_ITERATE(Parts, residence, rit) if (*rit != self) { - PCU_COMM_PACK(*rit, gid); - PCU_COMM_PACK(*rit, vert); + m->getPCU()->Pack(*rit, gid); + m->getPCU()->Pack(*rit, vert); } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); MeshEntity* remote; - PCU_COMM_UNPACK(remote); - int from = PCU_Comm_Sender(); + m->getPCU()->Unpack(remote); + int from = m->getPCU()->Sender(); MeshEntity* vert = globalToVert[gid]; m->addRemote(vert, from, remote); } // who is not stuck? - lion_eprint(1, "%d done inside remotes \n",PCU_Comm_Self()); + lion_eprint(1, "%d done inside remotes \n",m->getPCU()->Self()); } NewElements assemble(Mesh2* m, const Gid* conn, int nelem, int etype, @@ -196,7 +195,7 @@ NewElements assemble(Mesh2* m, const Gid* conn, int nelem, int etype, void finalise(Mesh2* m, GlobalToVert& globalToVert) { constructResidence(m, globalToVert); - lion_eprint(1, "%d after residence \n",PCU_Comm_Self()); + lion_eprint(1, "%d after residence \n",m->getPCU()->Self()); constructRemotes(m, globalToVert); stitchMesh(m); m->acceptChanges(); @@ -213,13 +212,13 @@ NewElements construct(Mesh2* m, const Gid* conn, int nelem, int etype, void setCoords(Mesh2* m, const double* coords, int nverts, GlobalToVert& globalToVert) { - Gid max = getMax(globalToVert); + Gid max = getMax(globalToVert, m->getPCU()); Gid total = max + 1; - int peers = PCU_Comm_Peers(); + int peers = m->getPCU()->Peers(); int quotient = total / peers; int remainder = total % peers; int mySize = quotient; - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); if (self == (peers - 1)) mySize += remainder; Gid myOffset = (long)self * quotient; @@ -228,9 +227,9 @@ void setCoords(Mesh2* m, const double* coords, int nverts, This means we might need to send and recv some coords */ double* c = new double[mySize*3]; - Gid start = PCU_Exscan_Long(nverts); + Gid start = m->getPCU()->Exscan(nverts); - PCU_Comm_Begin(); // the forced 64 bit math below may not be necessary + m->getPCU()->Begin(); // the forced 64 bit math below may not be necessary Gid tmpL=start / quotient; int tmpInt=tmpL; int to = std::min(peers - 1, tmpInt); @@ -239,9 +238,9 @@ void setCoords(Mesh2* m, const double* coords, int nverts, int n = std::min(tmpInt, nverts); while (nverts > 0) { - PCU_COMM_PACK(to, start); - PCU_COMM_PACK(to, n); - PCU_Comm_Pack(to, coords, n*3*sizeof(double)); + m->getPCU()->Pack(to, start); + m->getPCU()->Pack(to, n); + m->getPCU()->Pack(to, coords, n*3*sizeof(double)); nverts -= n; start += n; @@ -249,51 +248,51 @@ void setCoords(Mesh2* m, const double* coords, int nverts, to = std::min(peers - 1, to + 1); n = std::min(quotient, nverts); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { - PCU_COMM_UNPACK(start); - PCU_COMM_UNPACK(n); // |||||| more in-place 64 bit math - PCU_Comm_Unpack(&c[(start - myOffset) * 3], n*3*sizeof(double)); + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { + m->getPCU()->Unpack(start); + m->getPCU()->Unpack(n); // |||||| more in-place 64 bit math + m->getPCU()->Unpack(&c[(start - myOffset) * 3], n*3*sizeof(double)); } /* Tell all the owners of the coords what we need */ typedef std::vector< std::vector > TmpParts; TmpParts tmpParts(mySize); - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_CONST_ITERATE(GlobalToVert, globalToVert, it) { Gid gid = it->first; Gid tmpL=gid / quotient; int tmpInt=tmpL; int to = std::min(peers - 1, tmpInt); - PCU_COMM_PACK(to, gid); + m->getPCU()->Pack(to, gid); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); - Gid from = PCU_Comm_Sender(); + m->getPCU()->Unpack(gid); + Gid from = m->getPCU()->Sender(); Gid tmpL=gid - myOffset; int tmpInt=tmpL; tmpParts.at(tmpInt).push_back(from); } /* Send the coords to everybody who want them */ - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int i = 0; i < mySize; ++i) { std::vector& parts = tmpParts[i]; for (size_t j = 0; j < parts.size(); ++j) { int to = parts[j]; Gid gid = i + myOffset; - PCU_COMM_PACK(to, gid); - PCU_Comm_Pack(to, &c[i*3], 3*sizeof(double)); + m->getPCU()->Pack(to, gid); + m->getPCU()->Pack(to, &c[i*3], 3*sizeof(double)); } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); double v[3]; - PCU_Comm_Unpack(v, sizeof(v)); + m->getPCU()->Unpack(v, sizeof(v)); Vector3 vv(v); m->setPoint(globalToVert[gid], 0, vv); } @@ -304,13 +303,13 @@ void setCoords(Mesh2* m, const double* coords, int nverts, void setMatches(Mesh2* m, const Gid* matches, int nverts, GlobalToVert& globalToVert) { - Gid max = getMax(globalToVert); + Gid max = getMax(globalToVert, m->getPCU()); Gid total = max + 1; - int peers = PCU_Comm_Peers(); + int peers = m->getPCU()->Peers(); int quotient = total / peers; int remainder = total % peers; int mySize = quotient; - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); if (self == (peers - 1)) mySize += remainder; Gid myOffset = (long)self * quotient; @@ -318,9 +317,9 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, /* Force each peer to have exactly mySize verts. This means we might need to send and recv some matches */ Gid* c = new Gid[mySize]; - Gid start = PCU_Exscan_Long(nverts); + Gid start = m->getPCU()->Exscan(nverts); - PCU_Comm_Begin(); + m->getPCU()->Begin(); Gid tmpL=start / quotient; int tmpInt=tmpL; @@ -329,9 +328,9 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, tmpInt=tmpL; int n = std::min(tmpInt, nverts); while (nverts > 0) { - PCU_COMM_PACK(to, start); - PCU_COMM_PACK(to, n); - PCU_Comm_Pack(to, matches, n*sizeof(Gid)); + m->getPCU()->Pack(to, start); + m->getPCU()->Pack(to, n); + m->getPCU()->Pack(to, matches, n*sizeof(Gid)); nverts -= n; start += n; @@ -339,51 +338,51 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, to = std::min(peers - 1, to + 1); n = std::min(quotient, nverts); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { - PCU_COMM_UNPACK(start); - PCU_COMM_UNPACK(n); /// in-place 64 - PCU_Comm_Unpack(&c[(start - myOffset)], n*sizeof(Gid)); + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { + m->getPCU()->Unpack(start); + m->getPCU()->Unpack(n); /// in-place 64 + m->getPCU()->Unpack(&c[(start - myOffset)], n*sizeof(Gid)); } /* Tell all the owners of the matches what we need */ typedef std::vector< std::vector > TmpParts; TmpParts tmpParts(mySize); - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_CONST_ITERATE(GlobalToVert, globalToVert, it) { Gid gid = it->first; int tmpI=gid / quotient; int to = std::min(peers - 1,tmpI); - PCU_COMM_PACK(to, gid); + m->getPCU()->Pack(to, gid); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); - int from = PCU_Comm_Sender(); + m->getPCU()->Unpack(gid); + int from = m->getPCU()->Sender(); tmpParts.at(gid - myOffset).push_back(from); } MeshTag* matchGidTag = m->createLongTag("matchGids", 1); /* Send the matches to everybody who wants them */ - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int i = 0; i < mySize; ++i) { std::vector& parts = tmpParts[i]; for (size_t j = 0; j < parts.size(); ++j) { int to = parts[j]; Gid gid = i + myOffset; Gid matchGid = c[i]; - PCU_COMM_PACK(to, gid); - PCU_COMM_PACK(to, matchGid); + m->getPCU()->Pack(to, gid); + m->getPCU()->Pack(to, matchGid); } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); Gid match; - PCU_COMM_UNPACK(match); + m->getPCU()->Unpack(match); PCU_ALWAYS_ASSERT(gid != match); PCU_ALWAYS_ASSERT(globalToVert.count(gid)); m->setLongTag(globalToVert[gid], matchGidTag, &match); @@ -394,25 +393,25 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, * the entity pointers and owners for mesh vertex gid [0..quotient), * process 1 holds gids [quotient..2*quotient), ... */ - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_CONST_ITERATE(GlobalToVert, globalToVert, it) { MeshEntity* e = it->second; // KEJ does not follow this Gid gid = it->first; int tmpI=gid / quotient; int to = std::min(peers - 1,tmpI); - PCU_COMM_PACK(to, gid); - PCU_COMM_PACK(to, e); + m->getPCU()->Pack(to, gid); + m->getPCU()->Pack(to, e); } - PCU_Comm_Send(); + m->getPCU()->Send(); typedef std::pair< int, apf::MeshEntity* > EntOwnerPtrs; typedef std::map< Gid, std::vector< EntOwnerPtrs > > GidPtrs; GidPtrs gidPtrs; - while (PCU_Comm_Receive()) { + while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); MeshEntity* vert; - PCU_COMM_UNPACK(vert); - int owner = PCU_Comm_Sender(); + m->getPCU()->Unpack(vert); + int owner = m->getPCU()->Sender(); gidPtrs[gid-myOffset].push_back(EntOwnerPtrs(owner,vert)); } @@ -420,7 +419,7 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, typedef std::pair MatchingPair; typedef std::map< MatchingPair, std::vector > MatchMap; MatchMap matchParts; - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_CONST_ITERATE(GlobalToVert, globalToVert, it) { //loop over local verts Gid gid = it->first; Gid matchGid; @@ -428,24 +427,24 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, if( matchGid != -1 ) { // marker for an unmatched vertex int tmpI=matchGid / quotient; int to = std::min(peers - 1,tmpI); //broker - PCU_COMM_PACK(to, gid); // send the local vert gid - PCU_COMM_PACK(to, matchGid); // and the match gid needed + m->getPCU()->Pack(to, gid); // send the local vert gid + m->getPCU()->Pack(to, matchGid); // and the match gid needed } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); // request from entity gid + m->getPCU()->Unpack(gid); // request from entity gid Gid matchGid; - PCU_COMM_UNPACK(matchGid); // requesting matched entity gid + m->getPCU()->Unpack(matchGid); // requesting matched entity gid MatchingPair mp(gid,matchGid); - int from = PCU_Comm_Sender(); + int from = m->getPCU()->Sender(); matchParts[mp].push_back(from); // store a list of the proceses that need the pair (entity gid, match gid) } /* Send the match pointer and owner process to everybody * who wants them */ - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(MatchMap,matchParts,it) { MatchingPair mp = it->first; Gid gid = mp.first; @@ -453,32 +452,32 @@ void setMatches(Mesh2* m, const Gid* matches, int nverts, std::vector parts = it->second; for(size_t i=0; igetPCU()->Pack(to, gid); + m->getPCU()->Pack(to, matchGid); size_t numMatches = gidPtrs[matchGid-myOffset].size(); - PCU_COMM_PACK(to, numMatches); + m->getPCU()->Pack(to, numMatches); for( size_t i=0; igetPCU()->Pack(to, owner); apf::MeshEntity* ent = eop.second; - PCU_COMM_PACK(to, ent); + m->getPCU()->Pack(to, ent); } } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { Gid gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); Gid matchGid; - PCU_COMM_UNPACK(matchGid); + m->getPCU()->Unpack(matchGid); size_t numMatches; - PCU_COMM_UNPACK(numMatches); + m->getPCU()->Unpack(numMatches); for(size_t i=0; igetPCU()->Unpack(owner); MeshEntity* match; - PCU_COMM_UNPACK(match); + m->getPCU()->Unpack(match); PCU_ALWAYS_ASSERT(globalToVert.count(gid)); MeshEntity* partner = globalToVert[gid]; PCU_ALWAYS_ASSERT(! (match == partner && owner == self) ); diff --git a/apf/apfConvert.cc b/apf/apfConvert.cc index 70ec499f9..4c2d2c0ac 100644 --- a/apf/apfConvert.cc +++ b/apf/apfConvert.cc @@ -1,4 +1,3 @@ -#include #include "apf.h" #include "apfConvert.h" #include "apfMesh.h" @@ -161,7 +160,7 @@ class Converter /* creating links backwards is OK because they go both ways; as long as every link gets a backward copy they will all get copies */ - PCU_Comm_Begin(); + inMesh->getPCU()->Begin(); MeshIterator* it = inMesh->begin(dim); MeshEntity *oldLeft; while ((oldLeft = inMesh->iterate(it))) @@ -173,23 +172,23 @@ class Converter { int rightPart = it->first; MeshEntity* oldRight = it->second; - PCU_COMM_PACK(rightPart,oldRight); - PCU_COMM_PACK(rightPart,newLeft); + inMesh->getPCU()->Pack(rightPart,oldRight); + inMesh->getPCU()->Pack(rightPart,newLeft); } } inMesh->end(it); - PCU_Comm_Send(); + inMesh->getPCU()->Send(); /* accumulation of residence sets... we could also constantly reclassify... */ std::map > map_ent; - while (PCU_Comm_Listen()) + while (inMesh->getPCU()->Listen()) { - int leftPart = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) + int leftPart = inMesh->getPCU()->Sender(); + while ( ! inMesh->getPCU()->Unpacked()) { MeshEntity* oldRight; - PCU_COMM_UNPACK(oldRight); + inMesh->getPCU()->Unpack(oldRight); MeshEntity* newLeft; - PCU_COMM_UNPACK(newLeft); + inMesh->getPCU()->Unpack(newLeft); MeshEntity *newRight = newFromOld[oldRight]; outMesh->addRemote(newRight, leftPart, newLeft); map_ent[newRight].push_back(leftPart); @@ -205,7 +204,7 @@ class Converter Copies remotes; Parts parts; parts.insert(vecEnt.begin(), vecEnt.end()); - parts.insert(PCU_Comm_Self()); + parts.insert(inMesh->getPCU()->Self()); outMesh->setResidence(newE, parts); } } @@ -407,7 +406,7 @@ class Converter { if (inMesh->getShape() != getLagrange(2) && inMesh->getShape() != getSerendipity()) return; - if ( ! PCU_Comm_Self()) + if ( ! inMesh->getPCU()->Self()) lion_eprint(1,"transferring quadratic mesh\n"); changeMeshShape(outMesh,inMesh->getShape(),/*project=*/false); convertField(inMesh->getCoordinateField(),outMesh->getCoordinateField()); @@ -415,7 +414,7 @@ class Converter void createMatches(int dim) { /* see createRemotes for the algorithm comments */ - PCU_Comm_Begin(); + inMesh->getPCU()->Begin(); MeshIterator* it = inMesh->begin(dim); MeshEntity *oldLeft; while ((oldLeft = inMesh->iterate(it))) @@ -427,21 +426,21 @@ class Converter { int rightPart = matches[i].peer; MeshEntity* oldRight = matches[i].entity; - PCU_COMM_PACK(rightPart,oldRight); - PCU_COMM_PACK(rightPart,newLeft); + inMesh->getPCU()->Pack(rightPart,oldRight); + inMesh->getPCU()->Pack(rightPart,newLeft); } } inMesh->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) + inMesh->getPCU()->Send(); + while (inMesh->getPCU()->Listen()) { - int leftPart = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) + int leftPart = inMesh->getPCU()->Sender(); + while ( ! inMesh->getPCU()->Unpacked()) { MeshEntity* oldRight; - PCU_COMM_UNPACK(oldRight); + inMesh->getPCU()->Unpack(oldRight); MeshEntity* newLeft; - PCU_COMM_UNPACK(newLeft); + inMesh->getPCU()->Unpack(newLeft); MeshEntity *newRight = newFromOld[oldRight]; outMesh->addMatch(newRight, leftPart, newLeft); } diff --git a/apf/apfConvertTags.h b/apf/apfConvertTags.h index 069caa43b..0cbf63ba2 100644 --- a/apf/apfConvertTags.h +++ b/apf/apfConvertTags.h @@ -1,17 +1,16 @@ #ifndef APF_CONVERT_TAGS_H #define APF_CONVERT_TAGS_H -#include #include "apf.h" #include "apfConvert.h" namespace { - static apf::Gid getMax(const apf::GlobalToVert& globalToVert) + static apf::Gid getMax(const apf::GlobalToVert& globalToVert, pcu::PCU *pcu_obj) { apf::Gid max = -1; APF_CONST_ITERATE(apf::GlobalToVert, globalToVert, it) max = std::max(max, it->first); - return PCU_Max_Long(max); // this is type-dependent + return pcu_obj->Max(max); // this is type-dependent } template inline @@ -72,13 +71,13 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, const T* vals, const int entries, int nverts, GlobalToVert& globalToVert) { - apf::Gid max = getMax(globalToVert); + apf::Gid max = getMax(globalToVert, m->getPCU()); apf::Gid total = max + 1; - int peers = PCU_Comm_Peers(); + int peers = m->getPCU()->Peers(); int quotient = total / peers; int remainder = total % peers; int mySize = quotient; - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); if (self == (peers - 1)) mySize += remainder; apf::Gid myOffset = (long)self * quotient; @@ -87,9 +86,9 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, This means we might need to send and recv some coords */ T* c = new T[mySize*entries]; - apf::Gid start = PCU_Exscan_Long(nverts); + apf::Gid start = m->getPCU()->Exscan(nverts); - PCU_Comm_Begin(); + m->getPCU()->Begin(); apf::Gid tmpL=start / quotient; int tmpI=tmpL; int to = std::min(peers - 1, tmpI); @@ -97,9 +96,9 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, tmpI=tmpL; int n = std::min(tmpI, nverts); while (nverts > 0) { - PCU_COMM_PACK(to, start); - PCU_COMM_PACK(to, n); - PCU_Comm_Pack(to, vals, n*entries*sizeof(T)); + m->getPCU()->Pack(to, start); + m->getPCU()->Pack(to, n); + m->getPCU()->Pack(to, vals, n*entries*sizeof(T)); nverts -= n; start += n; @@ -107,51 +106,51 @@ apf::MeshTag* setMappedTag(Mesh2* m, const char* tagName, to = std::min(peers - 1, to + 1); n = std::min(quotient, nverts); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { - PCU_COMM_UNPACK(start); - PCU_COMM_UNPACK(n); - PCU_Comm_Unpack(&c[(start - myOffset) * entries], n*entries*sizeof(T)); + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { + m->getPCU()->Unpack(start); + m->getPCU()->Unpack(n); + m->getPCU()->Unpack(&c[(start - myOffset) * entries], n*entries*sizeof(T)); } /* Tell all the owners of the data what we need */ typedef std::vector< std::vector > TmpParts; TmpParts tmpParts(mySize); - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_CONST_ITERATE(GlobalToVert, globalToVert, it) { apf::Gid gid = it->first; tmpL=gid / quotient; tmpI=tmpL; int to = std::min(peers - 1, tmpI); // replaced int to = std::min(peers - 1, gid / quotient); - PCU_COMM_PACK(to, gid); + m->getPCU()->Pack(to, gid); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { apf::Gid gid; - PCU_COMM_UNPACK(gid); - int from = PCU_Comm_Sender(); + m->getPCU()->Unpack(gid); + int from = m->getPCU()->Sender(); tmpParts.at(gid - myOffset).push_back(from); } /* Send the data to everybody who want them */ - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int i = 0; i < mySize; ++i) { std::vector& parts = tmpParts[i]; for (size_t j = 0; j < parts.size(); ++j) { int to = parts[j]; apf::Gid gid = i + myOffset; - PCU_COMM_PACK(to, gid); - PCU_Comm_Pack(to, &c[i*entries], entries*sizeof(T)); + m->getPCU()->Pack(to, gid); + m->getPCU()->Pack(to, &c[i*entries], entries*sizeof(T)); } } - PCU_Comm_Send(); + m->getPCU()->Send(); apf::MeshTag* t = createTag(m,tagName,entries); T* v = new T[entries]; - while (PCU_Comm_Receive()) { + while (m->getPCU()->Receive()) { apf::Gid gid; - PCU_COMM_UNPACK(gid); - PCU_Comm_Unpack(v, entries*sizeof(T)); + m->getPCU()->Unpack(gid); + m->getPCU()->Unpack(v, entries*sizeof(T)); setEntTag(m,t,globalToVert[gid],v); } delete [] v; diff --git a/apf/apfFieldData.cc b/apf/apfFieldData.cc index 2c3f1156a..db1f7dec0 100644 --- a/apf/apfFieldData.cc +++ b/apf/apfFieldData.cc @@ -1,4 +1,3 @@ -#include #include "apfFieldData.h" #include "apfShape.h" #include @@ -33,7 +32,7 @@ void synchronizeFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr) continue; MeshEntity* e; MeshIterator* it = m->begin(d); - PCU_Comm_Begin(); + m->getPCU()->Begin(); while ((e = m->iterate(it))) { if (( ! data->hasEntity(e))|| @@ -46,26 +45,26 @@ void synchronizeFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr) shr->getCopies(e, copies); for (size_t i = 0; i < copies.getSize(); ++i) { - PCU_COMM_PACK(copies[i].peer, copies[i].entity); - PCU_Comm_Pack(copies[i].peer, &(values[0]), n*sizeof(T)); + m->getPCU()->Pack(copies[i].peer, copies[i].entity); + m->getPCU()->Pack(copies[i].peer, &(values[0]), n*sizeof(T)); } apf::Copies ghosts; if (m->getGhosts(e, ghosts)) APF_ITERATE(Copies, ghosts, it) { - PCU_COMM_PACK(it->first, it->second); - PCU_Comm_Pack(it->first, &(values[0]), n*sizeof(T)); + m->getPCU()->Pack(it->first, it->second); + m->getPCU()->Pack(it->first, &(values[0]), n*sizeof(T)); } } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); int n = f->countValuesOn(e); NewArray values(n); - PCU_Comm_Unpack(&(values[0]),n*sizeof(T)); + m->getPCU()->Unpack(&(values[0]),n*sizeof(T)); data->set(e,&(values[0])); } } @@ -94,7 +93,7 @@ void reduceFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr, c MeshEntity* e; MeshIterator* it = m->begin(d); - PCU_Comm_Begin(); + m->getPCU()->Begin(); while ((e = m->iterate(it))) { /* send to all parts that can see this entity */ @@ -122,8 +121,8 @@ void reduceFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr, c for (size_t i = 0; i < copies.getSize(); ++i) { - PCU_COMM_PACK(copies[i].peer, copies[i].entity); - PCU_Comm_Pack(copies[i].peer, &(values[0]), n*sizeof(double)); + m->getPCU()->Pack(copies[i].peer, copies[i].entity); + m->getPCU()->Pack(copies[i].peer, &(values[0]), n*sizeof(double)); } // ghosts - only do them if this entity is on a partition boundary @@ -133,24 +132,24 @@ void reduceFieldData(FieldDataOf* data, Sharing* shr, bool delete_shr, c if (m->getGhosts(e, ghosts)) APF_ITERATE(Copies, ghosts, it2) { - PCU_COMM_PACK(it2->first, it2->second); - PCU_Comm_Pack(it2->first, &(values[0]), n*sizeof(double)); + m->getPCU()->Pack(it2->first, it2->second); + m->getPCU()->Pack(it2->first, &(values[0]), n*sizeof(double)); } } } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) - while ( ! PCU_Comm_Unpacked()) + m->getPCU()->Send(); + while (m->getPCU()->Listen()) + while ( ! m->getPCU()->Unpacked()) { /* receive and add. we only care about correctness on the owners */ MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); int n = f->countValuesOn(e); NewArray values(n); NewArray inValues(n); - PCU_Comm_Unpack(&(inValues[0]),n*sizeof(double)); + m->getPCU()->Unpack(&(inValues[0]),n*sizeof(double)); data->get(e,&(values[0])); for (int i = 0; i < n; ++i) values[i] = reduce_op.apply(values[i], inValues[i]); diff --git a/apf/apfH1Shapes.cc b/apf/apfH1Shapes.cc index 673b2b781..53bc4b243 100644 --- a/apf/apfH1Shapes.cc +++ b/apf/apfH1Shapes.cc @@ -15,7 +15,6 @@ #include #include #include -#include namespace apf { diff --git a/apf/apfIntegrate.cc b/apf/apfIntegrate.cc index bb93ad051..2925524c1 100644 --- a/apf/apfIntegrate.cc +++ b/apf/apfIntegrate.cc @@ -637,7 +637,7 @@ void Integrator::outElement() { } -void Integrator::parallelReduce() +void Integrator::parallelReduce(pcu::PCU*) { } @@ -656,7 +656,7 @@ void Integrator::process(Mesh* m, int d) destroyMeshElement(e); } m->end(elements); - this->parallelReduce(); + this->parallelReduce(m->getPCU()); } void Integrator::process(MeshElement* e) diff --git a/apf/apfL2Shapes.cc b/apf/apfL2Shapes.cc index 9f240b7d6..7c80095f3 100644 --- a/apf/apfL2Shapes.cc +++ b/apf/apfL2Shapes.cc @@ -15,7 +15,6 @@ #include #include #include -#include namespace apf { diff --git a/apf/apfMesh.cc b/apf/apfMesh.cc index 6db2d67d5..7f57fc983 100644 --- a/apf/apfMesh.cc +++ b/apf/apfMesh.cc @@ -158,7 +158,7 @@ Mesh::Type const Mesh::simplexTypes[4] = ,Mesh::TET }; -void Mesh::init(FieldShape* s) +void Mesh::init(FieldShape* s, pcu::PCU *PCUObj) { coordinateField = new VectorField(); FieldBase* baseP = coordinateField; @@ -166,6 +166,7 @@ void Mesh::init(FieldShape* s) baseP->init("coordinates",this,s,data); data->init(baseP); hasFrozenFields = false; + pcu_ = PCUObj; } Mesh::~Mesh() @@ -407,6 +408,12 @@ GlobalNumbering* Mesh::getGlobalNumbering(int i) return globalNumberings[i]; } +void Mesh::switchPCU(pcu::PCU *newPCU) +{ + PCU_ALWAYS_ASSERT(newPCU != nullptr); + pcu_ = newPCU; +} + void unite(Parts& into, Parts const& from) { @@ -865,8 +872,8 @@ void printTypes(Mesh* m) while ((e = m->iterate(it))) typeCnt[m->getType(e)]++; m->end(it); - PCU_Add_Longs(typeCnt,Mesh::TYPES); - if (!PCU_Comm_Self()) { + m->getPCU()->Add(typeCnt,Mesh::TYPES); + if (!m->getPCU()->Self()) { lion_oprint(1,"number of"); for (int i=0; igetPCU()->Add(n, 4); printTypes(m); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_oprint(1,"mesh entity counts: v %ld e %ld f %ld r %ld\n", n[0], n[1], n[2], n[3]); } @@ -892,8 +899,8 @@ void warnAboutEmptyParts(Mesh* m) int emptyParts = 0; if (!m->count(m->getDimension())) ++emptyParts; - emptyParts = PCU_Add_Int(emptyParts); - if (emptyParts && (!PCU_Comm_Self())) + emptyParts = m->getPCU()->Add(emptyParts); + if (emptyParts && (!m->getPCU()->Self())) lion_eprint(1,"APF warning: %d empty parts\n",emptyParts); } @@ -969,7 +976,7 @@ bool MatchedSharing::isLess(Copy const& a, Copy const& b) Copy MatchedSharing::getOwnerCopy(MeshEntity* e) { - Copy owner(PCU_Comm_Self(), e); + Copy owner(mesh->getPCU()->Self(), e); CopyArray copies; this->getCopies(e, copies); APF_ITERATE(CopyArray, copies, cit) @@ -987,7 +994,7 @@ int MatchedSharing::getOwner(MeshEntity* e) bool MatchedSharing::isOwned(MeshEntity* e) { Copy owner = this->getOwnerCopy(e); - return owner.peer == PCU_Comm_Self() && owner.entity == e; + return owner.peer == mesh->getPCU()->Self() && owner.entity == e; } void MatchedSharing::getCopies(MeshEntity* e, @@ -1009,23 +1016,23 @@ void MatchedSharing::getNeighbors(Parts& neighbors) neighbors.insert(cit->peer); } mesh->end(it); - neighbors.erase(PCU_Comm_Self()); + neighbors.erase(mesh->getPCU()->Self()); } void MatchedSharing::formCountMap() { size_t count = mesh->count(mesh->getDimension()); - countMap[PCU_Comm_Self()] = count; - PCU_Comm_Begin(); + countMap[mesh->getPCU()->Self()] = count; + mesh->getPCU()->Begin(); Parts neighbors; getNeighbors(neighbors); APF_ITERATE(Parts, neighbors, nit) - PCU_COMM_PACK(*nit, count); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + mesh->getPCU()->Pack(*nit, count); + mesh->getPCU()->Send(); + while (mesh->getPCU()->Receive()) { size_t oc; - PCU_COMM_UNPACK(oc); - countMap[PCU_Comm_Sender()] = oc; + mesh->getPCU()->Unpack(oc); + countMap[mesh->getPCU()->Sender()] = oc; } } @@ -1033,7 +1040,7 @@ bool MatchedSharing::isShared(MeshEntity* e) { CopyArray copies; this->getCopies(e, copies); APF_ITERATE(CopyArray, copies, it) - if (it->peer != PCU_Comm_Self()) + if (it->peer != mesh->getPCU()->Self()) return true; return false; } @@ -1167,20 +1174,20 @@ void getAlignment(Mesh* m, MeshEntity* elem, MeshEntity* boundary, rotate = findIn(bv, nbv, ebv[0]); } -void packString(std::string s, int to) +void packString(std::string s, int to, pcu::PCU *PCUObj) { size_t len = s.length(); - PCU_COMM_PACK(to, len); - PCU_Comm_Pack(to, s.c_str(), len); + PCUObj->Pack(to, len); + PCUObj->Pack(to, s.c_str(), len); } -std::string unpackString() +std::string unpackString(pcu::PCU *PCUObj) { std::string s; size_t len; - PCU_COMM_UNPACK(len); + PCUObj->Unpack(len); s.resize(len); - PCU_Comm_Unpack((void*)s.c_str(), len); + PCUObj->Unpack((void*)s.c_str(), len); return s; } @@ -1188,20 +1195,20 @@ void packTagInfo(Mesh* m, MeshTag* t, int to) { std::string name; name = m->getTagName(t); - packString(name, to); + packString(name, to, m->getPCU()); int type; type = m->getTagType(t); - PCU_COMM_PACK(to, type); + m->getPCU()->Pack(to, type); int size; size = m->getTagSize(t); - PCU_COMM_PACK(to, size); + m->getPCU()->Pack(to, size); } -void unpackTagInfo(std::string& name, int& type, int& size) +void unpackTagInfo(std::string& name, int& type, int& size, pcu::PCU *PCUObj) { - name = unpackString(); - PCU_COMM_UNPACK(type); - PCU_COMM_UNPACK(size); + name = unpackString(PCUObj); + PCUObj->Unpack(type); + PCUObj->Unpack(size); } char const* const dimName[4] = { diff --git a/apf/apfMesh.h b/apf/apfMesh.h index ac60009f5..d862333f2 100644 --- a/apf/apfMesh.h +++ b/apf/apfMesh.h @@ -14,6 +14,7 @@ #include #include #include +#include #include "apfVector.h" #include "apfDynamicArray.h" @@ -107,7 +108,7 @@ class Mesh \param s the field distribution of the coordinate field, apf::getLagrange(1) is a good default */ - void init(FieldShape* s); + void init(FieldShape* s, pcu::PCU *PCUObj); /** \brief destroy the base class structures. \details this does not destroy the underlying data structure, use apf::Mesh::destroyNative for that. @@ -395,6 +396,9 @@ class Mesh GlobalNumbering* findGlobalNumbering(const char* name); GlobalNumbering* getGlobalNumbering(int i); + /** \brief get the mesh pcu */ + pcu::PCU* getPCU() const {return pcu_;} + void switchPCU(pcu::PCU *newPCU); /** \brief true if any associated fields use array storage */ bool hasFrozenFields; protected: @@ -402,6 +406,7 @@ class Mesh std::vector fields; std::vector numberings; std::vector globalNumberings; + pcu::PCU* pcu_; }; /** \brief run consistency checks on an apf::Mesh structure @@ -615,10 +620,10 @@ int getFirstType(Mesh* m, int dim); void getAlignment(Mesh* m, MeshEntity* elem, MeshEntity* boundary, int& which, bool& flip, int& rotate); -void packString(std::string s, int to); -std::string unpackString(); +void packString(std::string s, int to, pcu::PCU *PCUObj); +std::string unpackString(pcu::PCU *PCUObj); void packTagInfo(Mesh* m, MeshTag* t, int to); -void unpackTagInfo(std::string& name, int& type, int& size); +void unpackTagInfo(std::string& name, int& type, int& size, pcu::PCU *PCUObj); extern char const* const dimName[4]; diff --git a/apf/apfMesh2.cc b/apf/apfMesh2.cc index 0f070e679..dfbca80aa 100644 --- a/apf/apfMesh2.cc +++ b/apf/apfMesh2.cc @@ -1,9 +1,7 @@ -#include #include #include "apfMesh2.h" #include "apfField.h" #include "apf.h" - #include "apfShape.h" #include "apfTagData.h" #include "apfNumbering.h" @@ -149,28 +147,28 @@ static void getCandidateParts(Mesh* m, MeshEntity* e, Parts& parts) static void packProposal(Mesh* m, MeshEntity* e, int to) { int t = m->getType(e); - PCU_COMM_PACK(to,t); - PCU_COMM_PACK(to,e); + m->getPCU()->Pack(to,t); + m->getPCU()->Pack(to,e); int d = getDimension(m, e); Downward down; int nd = m->getDownward(e, d - 1, down); - PCU_COMM_PACK(to,nd); + m->getPCU()->Pack(to,nd); for (int i = 0; i < nd; ++i) { Copies remotes; m->getRemotes(down[i], remotes); MeshEntity* dr = remotes[to]; - PCU_COMM_PACK(to,dr); + m->getPCU()->Pack(to,dr); } } -static void unpackProposal(int& t, MeshEntity*& e, Downward& da) +static void unpackProposal(int& t, MeshEntity*& e, Downward& da, pcu::PCU *PCUObj) { - PCU_COMM_UNPACK(t); - PCU_COMM_UNPACK(e); + PCUObj->Unpack(t); + PCUObj->Unpack(e); int nd; - PCU_COMM_UNPACK(nd); + PCUObj->Unpack(nd); for (int i=0; i < nd; ++i) - PCU_COMM_UNPACK(da[i]); + PCUObj->Unpack(da[i]); } void stitchMesh(Mesh2* m) @@ -179,7 +177,7 @@ void stitchMesh(Mesh2* m) int d_max = m->getDimension(); MeshEntity* e; for (int d=1; d < d_max; ++d) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); MeshIterator* it = m->begin(d); while ((e = m->iterate(it))) { Parts candidateParts; @@ -189,13 +187,13 @@ void stitchMesh(Mesh2* m) packProposal(m, e,*pit); } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { - int from = PCU_Comm_Sender(); - while (!PCU_Comm_Unpacked()) { + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { + int from = m->getPCU()->Sender(); + while (!m->getPCU()->Unpacked()) { int t; Downward da; - unpackProposal(t, e, da); + unpackProposal(t, e, da, m->getPCU()); MeshEntity* found = findUpward(m, t, da); if (found) m->addRemote(found, from, e); @@ -205,26 +203,26 @@ void stitchMesh(Mesh2* m) } } -static void packTagClone(Mesh2* m, MeshTag* t, int to) +static void packTagClone(Mesh2* m, MeshTag* t, int to, pcu::PCU *PCUObj) { std::string name; name = m->getTagName(t); - packString(name, to); + packString(name, to, PCUObj); int type; type = m->getTagType(t); - PCU_COMM_PACK(to, type); + PCUObj->Pack(to, type); int size; size = m->getTagSize(t); - PCU_COMM_PACK(to, size); + PCUObj->Pack(to, size); } static MeshTag* unpackTagClone(Mesh2* m) { - std::string name = unpackString(); + std::string name = unpackString(m->getPCU()); int type; - PCU_COMM_UNPACK(type); + m->getPCU()->Unpack(type); int size; - PCU_COMM_UNPACK(size); + m->getPCU()->Unpack(size); if (type == apf::Mesh::DOUBLE) return m->createDoubleTag(name.c_str(), size); if (type == apf::Mesh::INT) @@ -234,79 +232,84 @@ static MeshTag* unpackTagClone(Mesh2* m) return 0; } -static void packTagClones(Mesh2* m, int to) +static void packTagClones(Mesh2* m, int to, pcu::PCU *PCUObj) { DynamicArray tags; m->getTags(tags); int n = tags.getSize(); - PCU_COMM_PACK(to, n); + PCUObj->Pack(to, n); + /* warning! this loop goes backward to cater to MDS implementation-specific behavior. please forgive me. */ for (int i = n - 1; i >= 0; --i) - packTagClone(m, tags[i], to); + packTagClone(m, tags[i], to, PCUObj); } static void unpackTagClones(Mesh2* m) { int n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); for (int i = 0; i < n; ++i) unpackTagClone(m); } -static void packFieldClone(Field* f, int to) +static void packFieldClone(Field* f, int to, pcu::PCU *PCUObj) { std::string name = f->getName(); - packString(name, to); + packString(name, to, PCUObj); int valueType = f->getValueType(); - PCU_COMM_PACK(to, valueType); + PCUObj->Pack(to, valueType); + int components = f->countComponents(); - PCU_COMM_PACK(to, components); + PCUObj->Pack(to, components); + std::string shapeName = f->getShape()->getName(); - packString(shapeName, to); + packString(shapeName, to, PCUObj); + /* warning! this only supports tag-stored fields */ } static Field* unpackFieldClone(Mesh2* m) { - std::string name = unpackString(); + std::string name = unpackString(m->getPCU()); int valueType; - PCU_COMM_UNPACK(valueType); + m->getPCU()->Unpack(valueType); int components; - PCU_COMM_UNPACK(components); - std::string shapeName = unpackString(); + m->getPCU()->Unpack(components); + std::string shapeName = unpackString(m->getPCU()); FieldShape* shape = getShapeByName(shapeName.c_str()); PCU_ALWAYS_ASSERT(shape); /* warning! this only supports tag-stored fields */ return makeField(m, name.c_str(), valueType, components, shape, new TagDataOf); } -static void packFieldClones(Mesh2* m, int to) +static void packFieldClones(Mesh2* m, int to, pcu::PCU *PCUObj) { int n = m->countFields(); - PCU_COMM_PACK(to, n); + PCUObj->Pack(to, n); + for (int i = 0; i < n; ++i) - packFieldClone(m->getField(i), to); + packFieldClone(m->getField(i), to, PCUObj); } static void unpackFieldClones(Mesh2* m) { int n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); for (int i = 0; i < n; ++i) unpackFieldClone(m); } -static void packMeshShape(Mesh2* m, int to) +static void packMeshShape(Mesh2* m, int to, pcu::PCU *PCUObj) { std::string shapeName = m->getShape()->getName(); - packString(shapeName, to); + packString(shapeName, to, PCUObj); } static void unpackMeshShape(Mesh2* m) { - std::string shapeName = unpackString(); + std::string shapeName = unpackString(m->getPCU()); FieldShape* shape = getShapeByName(shapeName.c_str()); PCU_ALWAYS_ASSERT(shape); if (shape != m->getShape()) { @@ -314,11 +317,11 @@ static void unpackMeshShape(Mesh2* m) } } -void packDataClone(Mesh2* m, int to) +void packDataClone(Mesh2* m, int to, pcu::PCU *PCUObj) { - packTagClones(m, to); - packMeshShape(m, to); - packFieldClones(m, to); + packTagClones(m, to, PCUObj); + packMeshShape(m, to, PCUObj); + packFieldClones(m, to, PCUObj); } void unpackDataClone(Mesh2* m) diff --git a/apf/apfMesh2.h b/apf/apfMesh2.h index 7b5795ea2..2dda18e9b 100644 --- a/apf/apfMesh2.h +++ b/apf/apfMesh2.h @@ -136,7 +136,7 @@ void migrateSilent(Mesh2* m, Migration* plan); This function globally sets the limit on migration, which causes any migration requests greater than the limit to be performed as several consecutive migrations. */ -void setMigrationLimit(size_t maxElements); +void setMigrationLimit(size_t maxElements, pcu::PCU *PCUObj); class Field; @@ -199,13 +199,13 @@ void stitchMesh(Mesh2* m); /** \brief removes all entities and fields. */ void clear(Mesh2* m); -void packDataClone(Mesh2* m, int to); +void packDataClone(Mesh2* m, int to, pcu::PCU *PCUObj); void unpackDataClone(Mesh2* m); // common functions for migration/ghosting/distribution typedef std::vector EntityVector; -void packParts(int to, Parts& parts); -void unpackParts(Parts& parts); +void packParts(int to, Parts& parts, pcu::PCU *PCUObj); +void unpackParts(Parts& parts, pcu::PCU *PCUObj); void moveEntities( Mesh2* m, EntityVector senders[4]); @@ -220,7 +220,7 @@ void reduceMatchingToSenders( Mesh2* m, EntityVector senders[4]); void getSenders(Mesh2* m,EntityVector affected[4],EntityVector senders[4]); -void split(Copies& remotes, Parts& parts, Parts& newParts); +void split(Copies& remotes, Parts& parts, Parts& newParts, pcu::PCU *PCUObj); // seol void packEntity(Mesh2* m, int to, MeshEntity* e, DynamicArray& tags, bool ghosting=false); diff --git a/apf/apfMigrate.cc b/apf/apfMigrate.cc index a0df01481..d17d6da96 100644 --- a/apf/apfMigrate.cc +++ b/apf/apfMigrate.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include "apfMesh2.h" #include "apfCavityOp.h" #include "apf.h" @@ -28,7 +27,7 @@ static void getAffected( EntityVector affected[4]) { int maxDimension = m->getDimension(); - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); affected[maxDimension].reserve(plan->count()); for (int i=0; i < plan->count(); ++i) { @@ -43,7 +42,7 @@ static void getAffected( for (int dimension=maxDimension-1; dimension >= 0; --dimension) { int upDimension = dimension + 1; - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(EntityVector,affected[upDimension],it) { MeshEntity* up = *it; @@ -59,21 +58,21 @@ static void getAffected( Copies remotes; m->getRemotes(adjacent[i],remotes); APF_ITERATE(Copies,remotes,rit) - PCU_COMM_PACK(rit->first,rit->second); + m->getPCU()->Pack(rit->first,rit->second); if (m->hasMatching()) { Matches matches; m->getMatches(adjacent[i],matches); for (size_t j=0; j < matches.getSize(); ++j) - PCU_COMM_PACK(matches[j].peer,matches[j].entity); + m->getPCU()->Pack(matches[j].peer,matches[j].entity); } }//downward adjacent loop }//upward affected loop - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { MeshEntity* entity; - PCU_COMM_UNPACK(entity); + m->getPCU()->Unpack(entity); if ( ! m->hasTag(entity,tag)) { m->setIntTag(entity,tag,&dummy); @@ -117,7 +116,7 @@ void reduceMatchingToSenders( EntityVector senders[4]) { if ( ! m->hasMatching()) return; - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int d=0; d < 4; ++d) { for (size_t i=0; i < senders[d].size(); ++i) @@ -128,23 +127,23 @@ void reduceMatchingToSenders( for (size_t j=0; j < matches.getSize(); ++j) { /* advertise to the match that this is a sender */ int to = matches[j].peer; - PCU_COMM_PACK(to,matches[j].entity); - PCU_COMM_PACK(to,e); + m->getPCU()->Pack(to,matches[j].entity); + m->getPCU()->Pack(to,e); } /* now forget all other matchings */ m->clearMatches(e); } } - PCU_Comm_Send(); - while (PCU_Comm_Listen()) + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { - int sender = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) + int sender = m->getPCU()->Sender(); + while ( ! m->getPCU()->Unpacked()) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); MeshEntity* match; - PCU_COMM_UNPACK(match); + m->getPCU()->Unpack(match); /* all matches of the sender receive this. We only care that the senders themselves receive messages from only other senders matched to them, @@ -162,25 +161,25 @@ static Parts makeResidence(int part) return r; } -void packParts(int to, Parts& parts) +void packParts(int to, Parts& parts, pcu::PCU *PCUObj) { size_t n = parts.size(); - PCU_COMM_PACK(to,n); + PCUObj->Pack(to,n); APF_ITERATE(Parts,parts,it) { int p = *it; - PCU_COMM_PACK(to,p); + PCUObj->Pack(to,p); } } -void unpackParts(Parts& parts) +void unpackParts(Parts& parts, pcu::PCU *PCUObj) { size_t n; - PCU_COMM_UNPACK(n); + PCUObj->Unpack(n); for (size_t i=0;iUnpack(p); parts.insert(p); } } @@ -203,7 +202,7 @@ static void updateResidences( } for (int dimension = maxDimension-1; dimension >= 0; --dimension) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(EntityVector,affected[dimension],it) { MeshEntity* entity = *it; @@ -222,19 +221,19 @@ static void updateResidences( m->getRemotes(entity,remotes); APF_ITERATE(Copies,remotes,rit) { - PCU_COMM_PACK(rit->first,rit->second); - packParts(rit->first,newResidence); + m->getPCU()->Pack(rit->first,rit->second); + packParts(rit->first,newResidence, m->getPCU()); } } - PCU_Comm_Send(); - while(PCU_Comm_Receive()) + m->getPCU()->Send(); + while(m->getPCU()->Receive()) { MeshEntity* entity; - PCU_COMM_UNPACK(entity); + m->getPCU()->Unpack(entity); Parts current; m->getResidence(entity,current); Parts incoming; - unpackParts(incoming); + unpackParts(incoming, m->getPCU()); unite(current,incoming); m->setResidence(entity,current); } @@ -250,10 +249,11 @@ static void updateResidences( void split( Copies& remotes, Parts& parts, - Parts& newParts) + Parts& newParts, + pcu::PCU *PCUObj) { APF_ITERATE(Parts,parts,it) - if (( ! remotes.count(*it))&&(*it != PCU_Comm_Self())) + if (( ! remotes.count(*it))&&(*it != PCUObj->Self())) newParts.insert(*it); } @@ -262,15 +262,15 @@ static void packCommon( int to, MeshEntity* e) { - PCU_COMM_PACK(to,e); + m->getPCU()->Pack(to,e); ModelEntity* me = m->toModel(e); int modelType = m->getModelType(me); - PCU_COMM_PACK(to,modelType); + m->getPCU()->Pack(to,modelType); int modelTag = m->getModelTag(me); - PCU_COMM_PACK(to,modelTag); + m->getPCU()->Pack(to,modelTag); Parts residence; m->getResidence(e,residence); - packParts(to,residence); + packParts(to,residence, m->getPCU()); } void unpackCommon( @@ -279,12 +279,12 @@ void unpackCommon( ModelEntity*& c, Parts& residence) { - PCU_COMM_UNPACK(sender); + m->getPCU()->Unpack(sender); int modelType,modelTag; - PCU_COMM_UNPACK(modelType); - PCU_COMM_UNPACK(modelTag); + m->getPCU()->Unpack(modelType); + m->getPCU()->Unpack(modelTag); c = m->findModelEntity(modelType,modelTag); - unpackParts(residence); + unpackParts(residence, m->getPCU()); } static void packVertex( @@ -294,9 +294,9 @@ static void packVertex( { Vector3 p; m->getPoint(e,0,p); - PCU_COMM_PACK(to,p); + m->getPCU()->Pack(to,p); m->getParam(e,p); - PCU_COMM_PACK(to,p); + m->getPCU()->Pack(to,p); } MeshEntity* unpackVertex( @@ -304,9 +304,9 @@ MeshEntity* unpackVertex( ModelEntity* c) { Vector3 point; - PCU_COMM_UNPACK(point); + m->getPCU()->Unpack(point); Vector3 param; - PCU_COMM_UNPACK(param); + m->getPCU()->Unpack(param); return m->createVertex(c,point,param); } @@ -321,7 +321,7 @@ static void packReference( if (found!=remotes.end()) { MeshEntity* remote = found->second; - PCU_COMM_PACK(to,remote); + m->getPCU()->Pack(to,remote); } else { @@ -330,7 +330,7 @@ static void packReference( found = ghosts.find(to); PCU_ALWAYS_ASSERT(found!=ghosts.end()); MeshEntity* ghost = found->second; - PCU_COMM_PACK(to,ghost); + m->getPCU()->Pack(to,ghost); } } @@ -339,18 +339,19 @@ static void packDownward(Mesh2* m, int to, MeshEntity* e) Downward down; int d = getDimension(m, e); int n = m->getDownward(e,d-1,down); - PCU_COMM_PACK(to,n); + m->getPCU()->Pack(to,n); for (int i=0; i < n; ++i) packReference(m,to,down[i]); } static void unpackDownward( - Downward& entities) + Downward& entities, + pcu::PCU *PCUObj) { int n; - PCU_COMM_UNPACK(n); + PCUObj->Unpack(n); for (int i=0; i < n; ++i) - PCU_COMM_UNPACK(entities[i]); + PCUObj->Unpack(entities[i]); } static void packNonVertex( @@ -366,7 +367,7 @@ MeshEntity* unpackNonVertex( int type, ModelEntity* c) { Downward down; - unpackDownward(down); + unpackDownward(down, m->getPCU()); return m->createEntity(type,c,down); } @@ -381,26 +382,26 @@ static void packTags( for (size_t i=0; i < total; ++i) if (m->hasTag(e,tags[i])) ++n; - PCU_COMM_PACK(to,n); + m->getPCU()->Pack(to,n); for (size_t i=0; i < total; ++i) { MeshTag* tag = tags[i]; if (m->hasTag(e,tag)) { - PCU_COMM_PACK(to,i); + m->getPCU()->Pack(to,i); int type = m->getTagType(tag); int size = m->getTagSize(tag); if (type == Mesh2::DOUBLE) { DynamicArray d(size); m->getDoubleTag(e,tag,&(d[0])); - PCU_Comm_Pack(to,&(d[0]),size*sizeof(double)); + m->getPCU()->Pack(to,&(d[0]),size*sizeof(double)); } if (type == Mesh2::INT) { DynamicArray d(size); m->getIntTag(e,tag,&(d[0])); - PCU_Comm_Pack(to,&(d[0]),size*sizeof(int)); + m->getPCU()->Pack(to,&(d[0]),size*sizeof(int)); } } } @@ -415,26 +416,26 @@ void packRemotes( Copies remotes; m->getRemotes(e,remotes); size_t n = remotes.size(); - PCU_COMM_PACK(to,n); + m->getPCU()->Pack(to,n); APF_ITERATE(Copies,remotes,rit) { int p=rit->first; MeshEntity* remote = rit->second; - PCU_COMM_PACK(to,p); - PCU_COMM_PACK(to,remote); + m->getPCU()->Pack(to,p); + m->getPCU()->Pack(to,remote); } } void unpackRemotes(Mesh2* m, MeshEntity* e) { size_t n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); for (size_t i=0; i < n; ++i) { int p; - PCU_COMM_UNPACK(p); + m->getPCU()->Unpack(p); MeshEntity* r; - PCU_COMM_UNPACK(r); + m->getPCU()->Unpack(r); m->addRemote(e, p, r); } } @@ -445,26 +446,26 @@ void unpackTags( DynamicArray& tags) { size_t n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); PCU_ALWAYS_ASSERT_VERBOSE(n<=tags.size(), "A tag was created that does not exist on all processes."); for (size_t t=0; t < n; ++t) { size_t i; - PCU_COMM_UNPACK(i); + m->getPCU()->Unpack(i); MeshTag* tag = tags[i]; int type = m->getTagType(tag); int size = m->getTagSize(tag); if (type == Mesh2::DOUBLE) { DynamicArray d(size); - PCU_Comm_Unpack(&(d[0]),size*sizeof(double)); + m->getPCU()->Unpack(&(d[0]),size*sizeof(double)); m->setDoubleTag(e,tag,&(d[0])); } if (type == Mesh2::INT) { DynamicArray d(size); - PCU_Comm_Unpack(&(d[0]),size*sizeof(int)); + m->getPCU()->Unpack(&(d[0]),size*sizeof(int)); m->setIntTag(e,tag,&(d[0])); } } @@ -478,7 +479,7 @@ void packEntity( bool ghosting) { int type = m->getType(e); - PCU_COMM_PACK(to,type); + m->getPCU()->Pack(to,type); packCommon(m,to,e); if (type == Mesh::VERTEX) packVertex(m,to,e); @@ -493,9 +494,9 @@ static MeshEntity* unpackEntity( Mesh2* m, DynamicArray& tags) { - int from = PCU_Comm_Sender(); + int from = m->getPCU()->Sender(); int type; - PCU_COMM_UNPACK(type); + m->getPCU()->Unpack(type); MeshEntity* sender; ModelEntity* c; Parts residence; @@ -527,7 +528,7 @@ static void sendEntities( Parts residence; m->getResidence(entity,residence); Parts sendTo; - split(remotes,residence,sendTo); + split(remotes,residence,sendTo,m->getPCU()); APF_ITERATE(Parts,sendTo,sit) packEntity(m,*sit,entity,tags); } @@ -539,7 +540,7 @@ static void receiveEntities( EntityVector& received) { received.reserve(1024); - while (PCU_Comm_Receive()) + while (m->getPCU()->Receive()) received.push_back(unpackEntity(m,tags)); } @@ -557,22 +558,22 @@ static void echoRemotes( m->getRemotes(entity,temp); int from = temp.begin()->first; MeshEntity* sender = temp.begin()->second; - PCU_COMM_PACK(from,sender); - PCU_COMM_PACK(from,entity); + m->getPCU()->Pack(from,sender); + m->getPCU()->Pack(from,entity); } } static void receiveRemotes(Mesh2* m) { - while (PCU_Comm_Listen()) + while (m->getPCU()->Listen()) { - int from = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) + int from = m->getPCU()->Sender(); + while ( ! m->getPCU()->Unpacked()) { MeshEntity* sender; - PCU_COMM_UNPACK(sender); + m->getPCU()->Unpack(sender); MeshEntity* entity; - PCU_COMM_UNPACK(entity); + m->getPCU()->Unpack(entity); PCU_ALWAYS_ASSERT(entity); m->addRemote(sender, from, entity); } @@ -596,35 +597,37 @@ static void getNewCopies( APF_ITERATE(Copies,allRemotes,it) if (residence.count(it->first)) newCopies.insert(*it); - int rank = PCU_Comm_Self(); + int rank = m->getPCU()->Self(); if (residence.count(rank)) newCopies[rank]=e; } static void packCopies( int to, - Copies& copies) + Copies& copies, + pcu::PCU *PCUObj) { int n = copies.size(); - PCU_COMM_PACK(to,n); + PCUObj->Pack(to,n); APF_ITERATE(Copies,copies,it) { - PCU_COMM_PACK(to,it->first); - PCU_COMM_PACK(to,it->second); + PCUObj->Pack(to,it->first); + PCUObj->Pack(to,it->second); } } static void unpackCopies( - Copies& copies) + Copies& copies, + pcu::PCU *PCUObj) { int n; - PCU_COMM_UNPACK(n); + PCUObj->Unpack(n); for (int i=0; i < n; ++i) { int part; - PCU_COMM_UNPACK(part); + PCUObj->Unpack(part); MeshEntity* remote; - PCU_COMM_UNPACK(remote); + PCUObj->Unpack(remote); PCU_ALWAYS_ASSERT(remote); copies[part]=remote; } @@ -634,8 +637,8 @@ static void bcastRemotes( Mesh2* m, EntityVector& senders) { - PCU_Comm_Begin(); - int rank = PCU_Comm_Self(); + m->getPCU()->Begin(); + int rank = m->getPCU()->Self(); APF_ITERATE(EntityVector,senders,it) { MeshEntity* e = *it; @@ -645,19 +648,19 @@ static void bcastRemotes( getNewCopies(m,e,allRemotes,newCopies); APF_ITERATE(Copies,allRemotes,rit) { - PCU_COMM_PACK(rit->first,rit->second); - packCopies(rit->first,newCopies); + m->getPCU()->Pack(rit->first,rit->second); + packCopies(rit->first,newCopies, m->getPCU()); } newCopies.erase(rank); m->setRemotes(e,newCopies); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); Copies copies; - unpackCopies(copies); + unpackCopies(copies, m->getPCU()); copies.erase(rank); m->setRemotes(e,copies); } @@ -668,9 +671,9 @@ static void setupRemotes( EntityVector& received, EntityVector& senders) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); echoRemotes(m,received); - PCU_Comm_Send(); + m->getPCU()->Send(); receiveRemotes(m); bcastRemotes(m,senders); } @@ -684,9 +687,9 @@ void moveEntities( int maxDimension = m->getDimension(); for (int dimension = 0; dimension <= maxDimension; ++dimension) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); sendEntities(m,senders[dimension],tags); - PCU_Comm_Send(); + m->getPCU()->Send(); EntityVector received; receiveEntities(m,tags,received); setupRemotes(m,received,senders[dimension]); @@ -704,8 +707,8 @@ static void updateSenderMatching( EntityVector affected[4], EntityVector senders[4]) { - PCU_Comm_Begin(); - int self = PCU_Comm_Self(); + m->getPCU()->Begin(); + int self = m->getPCU()->Self(); for (int d=0; d < 4; ++d) { for (size_t i=0; i < senders[d].size(); ++i) @@ -720,14 +723,14 @@ static void updateSenderMatching( m->getResidence(e,residence); /* pack the remote copies to itself, then add itself to the copies if it remains */ - PCU_COMM_PACK(self,e); - packCopies(self,copies); + m->getPCU()->Pack(self,e); + packCopies(self,copies, m->getPCU()); if (residence.count(self)) copies[self]=e; for (size_t j=0; j < matches.getSize(); ++j) { /* pack all copies to matched senders */ - PCU_COMM_PACK(matches[j].peer,matches[j].entity); - packCopies(matches[j].peer,copies); + m->getPCU()->Pack(matches[j].peer,matches[j].entity); + packCopies(matches[j].peer,copies, m->getPCU()); } } /* destroy all matching in the entire affected set. @@ -736,13 +739,13 @@ static void updateSenderMatching( for (size_t i=0; i < affected[d].size(); ++i) m->clearMatches(affected[d][i]); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); Copies copies; - unpackCopies(copies); + unpackCopies(copies, m->getPCU()); APF_ITERATE(Copies,copies,cit) m->addMatch(e,cit->first,cit->second); } @@ -753,8 +756,8 @@ static void bcastMatching( Mesh2* m, EntityVector senders[4]) { - PCU_Comm_Begin(); - int self = PCU_Comm_Self(); + m->getPCU()->Begin(); + int self = m->getPCU()->Self(); for (int d=0; d < 4; ++d) { for (size_t i=0; i < senders[d].size(); ++i) @@ -777,29 +780,29 @@ static void bcastMatching( APF_ITERATE(Copies,remotes,rit) { int to = rit->first; - PCU_COMM_PACK(to,rit->second); + m->getPCU()->Pack(to,rit->second); size_t n = matches.getSize(); - PCU_COMM_PACK(to,n); + m->getPCU()->Pack(to,n); for (size_t j=0; j < n; ++j) { - PCU_COMM_PACK(to,matches[j].peer); - PCU_COMM_PACK(to,matches[j].entity); + m->getPCU()->Pack(to,matches[j].peer); + m->getPCU()->Pack(to,matches[j].entity); } } } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); Match match; size_t n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); for (size_t i=0; i < n; ++i) { - PCU_COMM_UNPACK(match.peer); - PCU_COMM_UNPACK(match.entity); + m->getPCU()->Unpack(match.peer); + m->getPCU()->Unpack(match.entity); if ( ! ((match.peer == self)&& (match.entity == e))) m->addMatch(e,match.peer,match.entity); @@ -826,7 +829,7 @@ void deleteOldEntities( Mesh2* m, EntityVector affected[4]) { - int rank = PCU_Comm_Self(); + int rank = m->getPCU()->Self(); int maxDimension = m->getDimension(); for (int d=maxDimension; d >= 0; --d) APF_ITERATE(EntityVector,affected[d],it) @@ -858,10 +861,10 @@ static void migrate1(Mesh2* m, Migration* plan) const size_t maxMigrationLimit = 10*1000*1000; static size_t migrationLimit = maxMigrationLimit; -void setMigrationLimit(size_t maxElements) +void setMigrationLimit(size_t maxElements, pcu::PCU *PCUObj) { if( maxElements >= maxMigrationLimit ) { - if(!PCU_Comm_Self()) + if(!PCUObj->Self()) lion_eprint(1, "ERROR requested migration limit exceeds" " %lu... exiting\n", maxMigrationLimit); abort(); @@ -883,7 +886,7 @@ static void migrate2(Mesh2* m, Migration* plan) } delete plan; size_t sent = 0; - while (PCU_Or(sent < tmp.size())) + while (m->getPCU()->Or(sent < tmp.size())) { plan = new Migration(m); size_t send = std::min(tmp.size() - sent, migrationLimit); @@ -896,7 +899,7 @@ static void migrate2(Mesh2* m, Migration* plan) void migrateSilent(Mesh2* m, Migration* plan) { - if (PCU_Or(static_cast(plan->count()) > migrationLimit)) + if (m->getPCU()->Or(static_cast(plan->count()) > migrationLimit)) migrate2(m, plan); else migrate1(m, plan); diff --git a/apf/apfMixedNumbering.cc b/apf/apfMixedNumbering.cc index 5cc1f354a..ae50bcf79 100644 --- a/apf/apfMixedNumbering.cc +++ b/apf/apfMixedNumbering.cc @@ -8,7 +8,6 @@ #include "apfNumbering.h" #include "apfNumberingClass.h" #include "apfShape.h" -#include #include #include #include @@ -228,8 +227,9 @@ static int number_ghost( static void globalize( int dofs, std::vector const& owned, - std::vector& global) { - long start = PCU_Exscan_Long(long(dofs)); + std::vector& global, + pcu::PCU *PCUObj) { + long start = PCUObj->Exscan(dofs); DynamicArray nodes; for (size_t f=0; f < global.size(); ++f) { getNodes(owned[f], nodes); @@ -315,10 +315,11 @@ int numberGhost( void makeGlobal( std::vector& owned, - std::vector& global) { + std::vector& global, + pcu::PCU *PCUObj) { int dofs = countDOFs(owned); create_global(owned, global); - globalize(dofs, owned, global); + globalize(dofs, owned, global, PCUObj); } } diff --git a/apf/apfMixedNumbering.h b/apf/apfMixedNumbering.h index a8429aa09..87494a5c4 100644 --- a/apf/apfMixedNumbering.h +++ b/apf/apfMixedNumbering.h @@ -69,7 +69,8 @@ int numberGhost( * \param global The output global numberings. */ void makeGlobal( std::vector& owned, - std::vector& global); + std::vector& global, + pcu::PCU *PCUObj); } diff --git a/apf/apfNedelec.cc b/apf/apfNedelec.cc index deb9b7d3b..5ef3b5065 100644 --- a/apf/apfNedelec.cc +++ b/apf/apfNedelec.cc @@ -15,7 +15,6 @@ #include #include #include -#include namespace apf { diff --git a/apf/apfNumbering.cc b/apf/apfNumbering.cc index 5d62935e4..345fb8701 100644 --- a/apf/apfNumbering.cc +++ b/apf/apfNumbering.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include "apfNumbering.h" #include "apfNumberingClass.h" #include "apfField.h" @@ -228,10 +227,14 @@ void synchronize(Numbering * n, Sharing* shr, bool delete_shr) struct NoSharing : public Sharing { - int getOwner(MeshEntity*) {return PCU_Comm_Self();} + NoSharing(pcu::PCU *PCUObj):pcu_obj{PCUObj}{PCU_ALWAYS_ASSERT(PCUObj != nullptr);} + int getOwner(MeshEntity*) {return pcu_obj->Self();} bool isOwned(MeshEntity*) {return true;} virtual void getCopies(MeshEntity*, CopyArray&) {} bool isShared(MeshEntity*) {return false;} + + private: + pcu::PCU *pcu_obj; }; Numbering* numberNodes( @@ -278,7 +281,7 @@ Numbering* numberOwnedDimension(Mesh* mesh, const char* name, int dim, Numbering* numberOverlapDimension(Mesh* mesh, const char* name, int dim) { FieldShape* s = getConstant(dim); - Sharing* shr = new NoSharing(); + Sharing* shr = new NoSharing(mesh->getPCU()); return numberNodes(mesh, name, s, shr, true); } @@ -291,7 +294,7 @@ Numbering* numberOverlapNodes(Mesh* mesh, const char* name, FieldShape* s) { if (!s) s = mesh->getShape(); - Sharing* shr = new NoSharing(); + Sharing* shr = new NoSharing(mesh->getPCU()); return numberNodes(mesh, name, s, shr, true); } @@ -401,20 +404,20 @@ static void synchronizeEntitySet( Mesh* m, EntitySet& set) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(EntitySet,set,it) if (m->isShared(*it)) { Copies remotes; m->getRemotes(*it,remotes); APF_ITERATE(Copies,remotes,rit) - PCU_COMM_PACK(rit->first,rit->second); + m->getPCU()->Pack(rit->first,rit->second); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); set.insert(e); } } @@ -532,9 +535,9 @@ int getElementNumbers(GlobalNumbering* n, MeshEntity* e, return n->getData()->getElementData(e,numbers); } -static long exscan(long x) +static long exscan(long x, pcu::PCU *PCUObj) { - return PCU_Exscan_Long(x); + return PCUObj->Exscan(x); } template @@ -557,26 +560,26 @@ class Globalizer : public FieldOp } return false; } - void run(NumberingOf* n) + void run(NumberingOf* n, pcu::PCU *PCUObj) { numbering = n; data = n->getData(); start = countFieldNodes(n); - start = exscan(start); + start = exscan(start, PCUObj); apply(n); } }; -static void globalize(GlobalNumbering* n) +static void globalize(GlobalNumbering* n, pcu::PCU *PCUObj) { Globalizer g; - g.run(n); + g.run(n, PCUObj); } -void globalize(Numbering* n) +void globalize(Numbering* n, pcu::PCU *PCUObj) { Globalizer g; - g.run(n); + g.run(n, PCUObj); } GlobalNumbering* makeGlobal(Numbering* n, bool destroy) @@ -606,7 +609,7 @@ GlobalNumbering* makeGlobal(Numbering* n, bool destroy) } if (destroy) apf::destroyNumbering(n); - globalize(gn); + globalize(gn, m->getPCU()); return gn; } diff --git a/apf/apfNumbering.h b/apf/apfNumbering.h index 36cf3f46d..bdfbcd91d 100644 --- a/apf/apfNumbering.h +++ b/apf/apfNumbering.h @@ -222,7 +222,7 @@ void getNodes(GlobalNumbering* n, DynamicArray& nodes); number the vertices and elements of a mesh */ MeshTag* reorder(Mesh* mesh, const char* name); -void globalize(Numbering* n); +void globalize(Numbering* n, pcu::PCU *PCUObj); /** \brief number all components by simple iteration */ int naiveOrder(Numbering * num, Sharing * sharing = NULL); diff --git a/apf/apfPolyBasis1D.cc b/apf/apfPolyBasis1D.cc index 9af87c6e9..1a86c6a79 100644 --- a/apf/apfPolyBasis1D.cc +++ b/apf/apfPolyBasis1D.cc @@ -11,7 +11,6 @@ #include #include #include -#include namespace apf { diff --git a/apf/apfVerify.cc b/apf/apfVerify.cc index 448b1ee41..a348fa40a 100644 --- a/apf/apfVerify.cc +++ b/apf/apfVerify.cc @@ -1,4 +1,3 @@ -#include #include "apfMesh.h" #include "apf.h" #include @@ -212,43 +211,45 @@ static Copies getAllCopies(Mesh* m, MeshEntity* e) { Copies c; m->getRemotes(e, c); - c[PCU_Comm_Self()] = e; + c[m->getPCU()->Self()] = e; return c; } -static void verifyAllCopies(Copies& a) +static void verifyAllCopies(Copies& a, pcu::PCU *PCUObj) { APF_ITERATE(Copies, a, it) { PCU_ALWAYS_ASSERT(it->first >= 0); - PCU_ALWAYS_ASSERT(it->first < PCU_Comm_Peers()); + PCU_ALWAYS_ASSERT(it->first < PCUObj->Peers()); } } static void packCopies( int to, - Copies& copies) + Copies& copies, + pcu::PCU *PCUObj) { int n = copies.size(); - PCU_COMM_PACK(to,n); + PCUObj->Pack(to,n); APF_ITERATE(Copies,copies,it) { - PCU_COMM_PACK(to,it->first); - PCU_COMM_PACK(to,it->second); + PCUObj->Pack(to,it->first); + PCUObj->Pack(to,it->second); } } static void unpackCopies( - Copies& copies) + Copies& copies, + pcu::PCU *PCUObj) { int n; - PCU_COMM_UNPACK(n); + PCUObj->Unpack(n); for (int i=0; i < n; ++i) { int part; - PCU_COMM_UNPACK(part); + PCUObj->Unpack(part); MeshEntity* remote; - PCU_COMM_UNPACK(remote); + PCUObj->Unpack(remote); PCU_ALWAYS_ASSERT(remote); copies[part]=remote; } @@ -259,22 +260,22 @@ static void sendAllCopies(Mesh* m, MeshEntity* e) Copies a; Copies r; a = getAllCopies(m, e); - verifyAllCopies(a); + verifyAllCopies(a, m->getPCU()); m->getRemotes(e, r); - PCU_ALWAYS_ASSERT(!r.count(PCU_Comm_Self())); + PCU_ALWAYS_ASSERT(!r.count(m->getPCU()->Self())); APF_ITERATE(Copies, r, it) { - PCU_COMM_PACK(it->first, it->second); - packCopies(it->first, a); + m->getPCU()->Pack(it->first, it->second); + packCopies(it->first, a, m->getPCU()); } } static void receiveAllCopies(Mesh* m) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); Copies a; - unpackCopies(a); + unpackCopies(a, m->getPCU()); Copies b = getAllCopies(m, e); PCU_ALWAYS_ASSERT(a == b); } @@ -283,15 +284,15 @@ static void verifyRemoteCopies(Mesh* m) { for (int d = 0; d <= m->getDimension(); ++d) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); MeshIterator* it = m->begin(d); MeshEntity* e; while ((e = m->iterate(it))) if (m->isShared(e) && !m->isGhost(e)) sendAllCopies(m, e); m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) receiveAllCopies(m); } } @@ -304,18 +305,18 @@ static void sendGhostCopies(Mesh* m, MeshEntity* e) PCU_ALWAYS_ASSERT(g.size()); APF_ITERATE(Copies, g, it) { - PCU_COMM_PACK(it->first, it->second); - PCU_COMM_PACK(it->first, e); + m->getPCU()->Pack(it->first, it->second); + m->getPCU()->Pack(it->first, e); } } static void receiveGhostCopies(Mesh* m) { - int from = PCU_Comm_Sender(); + int from = m->getPCU()->Sender(); MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); MeshEntity* g; - PCU_COMM_UNPACK(g); + m->getPCU()->Unpack(g); PCU_ALWAYS_ASSERT(m->isGhost(e) || m->isGhosted(e)); Copies ghosts; m->getGhosts(e,ghosts); @@ -331,7 +332,7 @@ static void receiveGhostCopies(Mesh* m) static void verifyGhostCopies(Mesh* m) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int d = 0; d <= m->getDimension(); ++d) { MeshIterator* it = m->begin(d); @@ -341,8 +342,8 @@ static void verifyGhostCopies(Mesh* m) sendGhostCopies(m, e); m->end(it); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) receiveGhostCopies(m); } @@ -359,25 +360,25 @@ static bool hasMatch( return false; } -static void sendSelfToMatches(MeshEntity* e, Matches& matches) +static void sendSelfToMatches(MeshEntity* e, Matches& matches, pcu::PCU *PCUObj) { APF_ITERATE(Matches, matches, it) { - PCU_ALWAYS_ASSERT(!((it->peer == PCU_Comm_Self())&&(it->entity == e))); - PCU_COMM_PACK(it->peer, e); - PCU_COMM_PACK(it->peer, it->entity); + PCU_ALWAYS_ASSERT(!((it->peer == PCUObj->Self())&&(it->entity == e))); + PCUObj->Pack(it->peer, e); + PCUObj->Pack(it->peer, it->entity); } } static void receiveMatches(Mesh* m) { MeshEntity* source; - PCU_COMM_UNPACK(source); + m->getPCU()->Unpack(source); MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); Matches matches; m->getMatches(e, matches); - PCU_ALWAYS_ASSERT(hasMatch(matches, PCU_Comm_Sender(), source)); + PCU_ALWAYS_ASSERT(hasMatch(matches, m->getPCU()->Sender(), source)); } static bool hasDuplicates(Matches const& matches) { @@ -391,7 +392,7 @@ static bool hasDuplicates(Matches const& matches) { static void verifyMatches(Mesh* m) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int d = 0; d <= m->getDimension(); ++d) { MeshIterator* it = m->begin(d); @@ -400,12 +401,12 @@ static void verifyMatches(Mesh* m) Matches matches; m->getMatches(e, matches); PCU_ALWAYS_ASSERT(!hasDuplicates(matches)); - sendSelfToMatches(e, matches); + sendSelfToMatches(e, matches, m->getPCU()); } m->end(it); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) receiveMatches(m); } @@ -419,9 +420,9 @@ static void sendCoords(Mesh* m, MeshEntity* e) m->getRemotes(e, r); APF_ITERATE(Copies, r, it) { - PCU_COMM_PACK(it->first, it->second); - PCU_COMM_PACK(it->first, x); - PCU_COMM_PACK(it->first, p); + m->getPCU()->Pack(it->first, it->second); + m->getPCU()->Pack(it->first, x); + m->getPCU()->Pack(it->first, p); } } @@ -430,9 +431,9 @@ static bool receiveCoords(Mesh* m) MeshEntity* e; Vector3 ox; Vector3 op; - PCU_COMM_UNPACK(e); - PCU_COMM_UNPACK(ox); - PCU_COMM_UNPACK(op); + m->getPCU()->Unpack(e); + m->getPCU()->Unpack(ox); + m->getPCU()->Unpack(op); Vector3 x; Vector3 p(0,0,0); m->getPoint(e, 0, x); @@ -443,19 +444,19 @@ static bool receiveCoords(Mesh* m) static long verifyCoords(Mesh* m) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); MeshIterator* it = m->begin(0); MeshEntity* e; while ((e = m->iterate(it))) if (m->isShared(e)) sendCoords(m, e); m->end(it); - PCU_Comm_Send(); + m->getPCU()->Send(); long n = 0; - while (PCU_Comm_Receive()) + while (m->getPCU()->Receive()) if (!receiveCoords(m)) ++n; - return PCU_Add_Long(n); + return m->getPCU()->Add(n); } long verifyVolumes(Mesh* m, bool printVolumes) @@ -481,12 +482,12 @@ long verifyVolumes(Mesh* m, bool printVolumes) } } m->end(it); - return PCU_Add_Long(n); + return m->getPCU()->Add(n); } static void packAlignment(Mesh* m, MeshEntity* e, MeshEntity* r, int to) { - PCU_COMM_PACK(to,r); + m->getPCU()->Pack(to,r); int d = getDimension(m, e); Downward down; int nd = m->getDownward(e, d - 1, down); @@ -494,7 +495,7 @@ static void packAlignment(Mesh* m, MeshEntity* e, MeshEntity* r, int to) Copies remotes; m->getRemotes(down[i], remotes); MeshEntity* dr = remotes[to]; - PCU_COMM_PACK(to,dr); + m->getPCU()->Pack(to,dr); } } @@ -509,12 +510,12 @@ static void sendAlignment(Mesh* m, MeshEntity* e) static void receiveAlignment(Mesh* m) { MeshEntity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); int d = getDimension(m, e); Downward down; int nd = m->getDownward(e, d - 1, down); for (int i = 0; i < nd; ++i) { - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); PCU_ALWAYS_ASSERT(down[i] == e); } } @@ -522,64 +523,64 @@ static void receiveAlignment(Mesh* m) static void verifyAlignment(Mesh* m) { for (int d = 1; d <= m->getDimension(); ++d) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); MeshIterator* it = m->begin(d); MeshEntity* e; while ((e = m->iterate(it))) if (m->isShared(e)) sendAlignment(m, e); m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) receiveAlignment(m); } } -void packFieldInfo(Field* f, int to) +void packFieldInfo(Field* f, int to, pcu::PCU *PCUObj) { std::string name; name = getName(f); - packString(name, to); + packString(name, to, PCUObj); int type, size; type = getValueType(f); - PCU_COMM_PACK(to, type); + PCUObj->Pack(to, type); size = countComponents(f); - PCU_COMM_PACK(to, size); + PCUObj->Pack(to, size); } -void unpackFieldInfo(std::string& name, int& type, int& size) +void unpackFieldInfo(std::string& name, int& type, int& size, pcu::PCU *PCUObj) { - name = unpackString(); - PCU_COMM_UNPACK(type); - PCU_COMM_UNPACK(size); + name = unpackString(PCUObj); + PCUObj->Unpack(type); + PCUObj->Unpack(size); } static void verifyFields(Mesh* m) { - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); int n = m->countFields(); std::vector fields; for (int i=0; igetField(i)); if (!fields.size()) return; - PCU_Comm_Begin(); + m->getPCU()->Begin(); if (self) { - PCU_COMM_PACK(self - 1, n); + m->getPCU()->Pack(self - 1, n); for (int i = 0; i < n; ++i) - packFieldInfo(fields[i], self - 1); + packFieldInfo(fields[i], self - 1, m->getPCU()); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { int n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); PCU_ALWAYS_ASSERT(fields.size() == (size_t)n); for (int i = 0; i < n; ++i) { std::string name; int type; int size; - unpackTagInfo(name, type, size); + unpackTagInfo(name, type, size, m->getPCU()); PCU_ALWAYS_ASSERT(name == getName(fields[i])); PCU_ALWAYS_ASSERT(type == getValueType(fields[i])); PCU_ALWAYS_ASSERT(size == countComponents(fields[i])); @@ -637,7 +638,7 @@ static void sendTagData(Mesh* m, MeshEntity* e, DynamicArray& tags, Co } default: break; } - PCU_Comm_Write(it->first, (void*)msg_send, msg_size); + m->getPCU()->Write(it->first, (void*)msg_send, msg_size); free(msg_send); } // apf_iterate } // for @@ -653,7 +654,7 @@ static void receiveTagData(Mesh* m, DynamicArray& tags) size_t msg_size; std::set mismatch_tags; - while(PCU_Comm_Read(&pid_from, &msg_recv, &msg_size)) + while(m->getPCU()->Read(&pid_from, &msg_recv, &msg_size)) { e = *((MeshEntity**)msg_recv); int *id = (int*)((char*)msg_recv+sizeof(MeshEntity*)); @@ -721,8 +722,8 @@ static void receiveTagData(Mesh* m, DynamicArray& tags) } // switch } // while - int global_size = PCU_Max_Int((int)mismatch_tags.size()); - if (global_size&&!PCU_Comm_Self()) + int global_size = m->getPCU()->Max((int)mismatch_tags.size()); + if (global_size&&!m->getPCU()->Self()) for (std::set::iterator it=mismatch_tags.begin(); it!=mismatch_tags.end(); ++it) lion_oprint(1," - tag \"%s\" data mismatch over remote/ghost copies\n", m->getTagName(*it)); } @@ -731,13 +732,13 @@ static void verifyTags(Mesh* m) { DynamicArray tags; m->getTags(tags); - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); int n = tags.getSize(); if (!n) return; - PCU_Comm_Begin(); + m->getPCU()->Begin(); if (self) { - PCU_COMM_PACK(self - 1, n); + m->getPCU()->Pack(self - 1, n); for (int i = 0; i < n; ++i) packTagInfo(m, tags[i], self - 1); } @@ -754,16 +755,16 @@ static void verifyTags(Mesh* m) lion_oprint(1,"\n"); } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { int n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); PCU_ALWAYS_ASSERT(tags.getSize() == (size_t)n); for (int i = 0; i < n; ++i) { std::string name; int type; int size; - unpackTagInfo(name, type, size); + unpackTagInfo(name, type, size, m->getPCU()); PCU_ALWAYS_ASSERT(name == m->getTagName(tags[i])); PCU_ALWAYS_ASSERT(type == m->getTagType(tags[i])); PCU_ALWAYS_ASSERT(size == m->getTagSize(tags[i])); @@ -774,12 +775,12 @@ static void verifyTags(Mesh* m) for (int d = 0; d <= m->getDimension(); ++d) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); MeshIterator* it = m->begin(d); MeshEntity* e; while ((e = m->iterate(it))) { - if (m->getOwner(e)!=PCU_Comm_Self()) continue; + if (m->getOwner(e)!=m->getPCU()->Self()) continue; if (m->isShared(e)) { Copies r; m->getRemotes(e, r); @@ -792,14 +793,14 @@ static void verifyTags(Mesh* m) } } // while m->end(it); - PCU_Comm_Send(); + m->getPCU()->Send(); receiveTagData(m, tags); } // for } void verify(Mesh* m, bool abort_on_error) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); verifyTags(m); verifyFields(m); @@ -828,13 +829,13 @@ void verify(Mesh* m, bool abort_on_error) verifyAlignment(m); verifyMatches(m); long n = verifyCoords(m); - if (n && (!PCU_Comm_Self())) + if (n && (!m->getPCU()->Self())) lion_eprint(1,"apf::verify fail: %ld coordinate mismatches\n", n); n = verifyVolumes(m); - if (n && (!PCU_Comm_Self())) + if (n && (!m->getPCU()->Self())) lion_eprint(1,"apf::verify warning: %ld negative simplex elements\n", n); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) lion_oprint(1,"mesh verified in %f seconds\n", t1 - t0); } diff --git a/apf/apfVtk.cc b/apf/apfVtk.cc index e71744a38..09e30ddb8 100644 --- a/apf/apfVtk.cc +++ b/apf/apfVtk.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include #include #include "apfMesh.h" @@ -46,7 +45,7 @@ static bool shouldPrint( bool isPrintable(FieldBase* f) { HasAll op; - return PCU_And(op.run(f)); + return f->getMesh()->getPCU()->And(op.run(f)); } static bool isNodal(FieldBase* f) @@ -254,9 +253,9 @@ static std::string getRelativePathPSource(int id) return ss.str(); } -static void writePSources(std::ostream& file) +static void writePSources(std::ostream& file, pcu::PCU *PCUObj) { - for (int i=0; i < PCU_Comm_Peers(); ++i) + for (int i=0; i < PCUObj->Peers(); ++i) { std::string fileName = stripPath(getPieceFileName(i)); std::string fileNameAndPath = getRelativePathPSource(i) + fileName; @@ -282,7 +281,7 @@ static void writePvtuFile(const char* prefix, writePPoints(file,m->getCoordinateField(),isWritingBinary); writePPointData(file,m,writeFields,isWritingBinary); writePCellData(file, m, writeFields, isWritingBinary, cellDim); - writePSources(file); + writePSources(file, m->getPCU()); file << "\n"; file << "\n"; } @@ -832,9 +831,9 @@ static void writeVtuFile(const char* prefix, bool isWritingBinary, int cellDim) { - double t0 = PCU_Time(); - std::string fileName = getPieceFileName(PCU_Comm_Self()); - std::string fileNameAndPath = getFileNameAndPathVtu(prefix, fileName, PCU_Comm_Self()); + double t0 = pcu::Time(); + std::string fileName = getPieceFileName(n->getMesh()->getPCU()->Self()); + std::string fileNameAndPath = getFileNameAndPathVtu(prefix, fileName, n->getMesh()->getPCU()->Self()); std::stringstream buf; Mesh* m = n->getMesh(); DynamicArray nodes; @@ -874,8 +873,8 @@ static void writeVtuFile(const char* prefix, buf << "\n"; buf << "\n"; buf << "\n"; - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) { lion_oprint(1,"writeVtuFile into buffers: %f seconds\n", t1 - t0); } @@ -884,8 +883,8 @@ static void writeVtuFile(const char* prefix, PCU_ALWAYS_ASSERT(file.is_open()); file << buf.rdbuf(); } - double t2 = PCU_Time(); - if (!PCU_Comm_Self()) + double t2 = pcu::Time(); + if (!m->getPCU()->Self()) { lion_oprint(1,"writeVtuFile buffers to disk: %f seconds\n", t2 - t1); } @@ -928,19 +927,19 @@ void writeVtkFilesRunner(const char* prefix, int cellDim) { if (cellDim == -1) cellDim = m->getDimension(); - double t0 = PCU_Time(); - if (!PCU_Comm_Self()) + double t0 = pcu::Time(); + if (!m->getPCU()->Self()) { safe_mkdir(prefix); - makeVtuSubdirectories(prefix, PCU_Comm_Peers()); + makeVtuSubdirectories(prefix, m->getPCU()->Peers()); writePvtuFile(prefix, m, writeFields, isWritingBinary, cellDim); } - PCU_Barrier(); + m->getPCU()->Barrier(); Numbering* n = numberOverlapNodes(m,"apf_vtk_number"); m->removeNumbering(n); writeVtuFile(prefix, n, writeFields, isWritingBinary, cellDim); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) { lion_oprint(1,"vtk files %s written in %f seconds\n", prefix, t1 - t0); } diff --git a/apf/apfVtkPieceWiseFields.cc b/apf/apfVtkPieceWiseFields.cc index 35016f775..d019cee2a 100644 --- a/apf/apfVtkPieceWiseFields.cc +++ b/apf/apfVtkPieceWiseFields.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include #include #include "apfMesh.h" @@ -72,7 +71,7 @@ static std::string getFileNameAndPathVtu(const char* prefix, static void writeNedelecVtkFile(const char* prefix, Mesh* m, std::vector writeFields) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); // get the number of points on this part MeshEntity* e; @@ -101,8 +100,8 @@ static void writeNedelecVtkFile(const char* prefix, Mesh* m, apf::Vector3(0., 0., 1.) }; - std::string fileName = getPieceFileName(PCU_Comm_Self()); - std::string fileNameAndPath = getFileNameAndPathVtu(prefix, fileName, PCU_Comm_Self()); + std::string fileName = getPieceFileName(m->getPCU()->Self()); + std::string fileNameAndPath = getFileNameAndPathVtu(prefix, fileName, m->getPCU()->Self()); std::stringstream buf; buf << "# vtk DataFile Version 3.0\n" @@ -163,7 +162,7 @@ static void writeNedelecVtkFile(const char* prefix, Mesh* m, buf << "LOOKUP_TABLE default\n"; it = m->begin(m->getDimension()); while( (e = m->iterate(it)) ) { - buf << PCU_Comm_Self() << '\n'; + buf << m->getPCU()->Self() << '\n'; } m->end(it); @@ -190,8 +189,8 @@ static void writeNedelecVtkFile(const char* prefix, Mesh* m, } m->end(it); } - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) { lion_oprint(1,"writeVtuFile into buffers: %f seconds\n", t1 - t0); } @@ -200,8 +199,8 @@ static void writeNedelecVtkFile(const char* prefix, Mesh* m, PCU_ALWAYS_ASSERT(file.is_open()); file << buf.rdbuf(); } - double t2 = PCU_Time(); - if (!PCU_Comm_Self()) + double t2 = pcu::Time(); + if (!m->getPCU()->Self()) { lion_oprint(1,"writeNedelecVtkFile buffers to disk: %f seconds\n", t2 - t1); } diff --git a/apf_cap/apfCAP.cc b/apf_cap/apfCAP.cc index 8e59797e8..31985c432 100644 --- a/apf_cap/apfCAP.cc +++ b/apf_cap/apfCAP.cc @@ -1,5 +1,4 @@ #include "apfCAP.h" -#include #include #include #include @@ -855,7 +854,7 @@ const char* MeshCAP::getTagName(MeshTag* tag) bool MeshCAP::isShared(MeshEntity* e) { (void)e; - if (PCU_Comm_Peers() != 1) + if (getPCU()->Peers() != 1) apf::fail("MeshCAP::isShared called in a parallel run!\n"); return false; } @@ -863,7 +862,7 @@ bool MeshCAP::isShared(MeshEntity* e) bool MeshCAP::isOwned(MeshEntity* e) { (void)e; - if (PCU_Comm_Peers() != 1) + if (getPCU()->Peers() != 1) apf::fail("MeshCAP::isOwned called in a parallel run!\n"); return true; } @@ -871,7 +870,7 @@ bool MeshCAP::isOwned(MeshEntity* e) int MeshCAP::getOwner(MeshEntity* e) { (void)e; - if (PCU_Comm_Peers() != 1) + if (getPCU()->Peers() != 1) apf::fail("MeshCAP::getOwner called in a parallel run!\n"); return 0; } @@ -880,23 +879,23 @@ void MeshCAP::getRemotes(MeshEntity* e, Copies& remotes) { (void)e; (void)remotes; - if (PCU_Comm_Peers() != 1) + if (getPCU()->Peers() != 1) apf::fail("MeshCAP::getRemotes called in a parallel run!\n"); } void MeshCAP::getResidence(MeshEntity* e, Parts& residence) { (void)e; - if (PCU_Comm_Peers() != 1) + if (getPCU()->Peers() != 1) apf::fail("MeshCAP::getResidence called in a parallel run!\n"); residence.insert(0); } int MeshCAP::getId() { - if (PCU_Comm_Peers() != 1) + if (getPCU()->Peers() != 1) apf::fail("MeshCAP::getId called in a parallel run!\n"); - return PCU_Comm_Self(); + return getPCU()->Self(); } void MeshCAP::migrate(Migration* plan) @@ -909,7 +908,7 @@ void MeshCAP::getMatches(MeshEntity* e, Matches& m) { (void)e; (void)m; - if (PCU_Comm_Peers() != 1) + if (getPCU()->Peers() != 1) apf::fail("MeshCAP::getMatches called in a parallel run!\n"); } @@ -935,10 +934,10 @@ MeshEntity* castEntity(capEntity* entity) return 0; } -Mesh2* createMesh(MeshDatabaseInterface* mdb, GeometryDatabaseInterface* gdb) +Mesh2* createMesh(MeshDatabaseInterface* mdb, GeometryDatabaseInterface* gdb, pcu::PCU *PCUObj) { MeshCAP* m = new MeshCAP(mdb, gdb); - m->init(getLagrange(1)); + m->init(getLagrange(1), PCUObj); return m; } diff --git a/apf_cap/apfCAP.h b/apf_cap/apfCAP.h index 102716bed..0c6279012 100644 --- a/apf_cap/apfCAP.h +++ b/apf_cap/apfCAP.h @@ -2,6 +2,7 @@ #define APFCAP #include +#include #include "CapstoneModule.h" #include "CreateMG_Framework_Geometry.h" @@ -24,7 +25,7 @@ namespace apf { * * \details This object should be destroyed by apf::destroyMesh. */ -Mesh2* createMesh(MeshDatabaseInterface* mdb, GeometryDatabaseInterface* gdb); +Mesh2* createMesh(MeshDatabaseInterface* mdb, GeometryDatabaseInterface* gdb, pcu::PCU *PCUObj); /** * \brief Casts a CapStone entity to an apf::MeshEntity. diff --git a/apf_sim/apfSIM.cc b/apf_sim/apfSIM.cc index 8132edfe7..09a447a19 100644 --- a/apf_sim/apfSIM.cc +++ b/apf_sim/apfSIM.cc @@ -1092,7 +1092,7 @@ static bool findMatches(Mesh* m) return found; } -Mesh2* createMesh(pParMesh mesh) +Mesh2* createMesh(pParMesh mesh, pcu::PCU *PCUObj) { /* require one part per process currently for SIM */ PCU_ALWAYS_ASSERT(PM_numParts(mesh)==1); @@ -1105,9 +1105,9 @@ Mesh2* createMesh(pParMesh mesh) serendipity = true; } if (serendipity) - m->init(getSerendipity()); + m->init(getSerendipity(), PCUObj); else - m->init(getLagrange(order)); + m->init(getLagrange(order), PCUObj); m->hasMatches = findMatches(m); return m; } diff --git a/apf_sim/apfSIM.h b/apf_sim/apfSIM.h index 3d87bd4eb..2a0c54969 100644 --- a/apf_sim/apfSIM.h +++ b/apf_sim/apfSIM.h @@ -16,7 +16,7 @@ namespace apf { * * \details This object should be destroyed by apf::destroyMesh. */ -Mesh2* createMesh(pParMesh mesh); +Mesh2* createMesh(pParMesh mesh, pcu::PCU *PCUObj); /** * \brief Casts a Simmetrix entity to an apf::MeshEntity. diff --git a/capstone_clis/capStone2VTK.cc b/capstone_clis/capStone2VTK.cc index 8c741f784..c6a8656c1 100644 --- a/capstone_clis/capStone2VTK.cc +++ b/capstone_clis/capStone2VTK.cc @@ -41,10 +41,11 @@ using namespace CreateMG::Geometry; int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); if (argc != 3) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -131,11 +132,11 @@ int main(int argc, char** argv) gmi_register_cap(); // convert the mesh to apf/mds mesh - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); apf::writeVtkFiles(folderName, mesh); gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneAnalyzeArea.cc b/capstone_clis/capStoneAnalyzeArea.cc index fdf4a0c90..6dd3332af 100644 --- a/capstone_clis/capStoneAnalyzeArea.cc +++ b/capstone_clis/capStoneAnalyzeArea.cc @@ -48,12 +48,13 @@ double addSizeDistribution(apf::Mesh2* m, int factor, int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - double initialTime = PCU_Time(); + double initialTime = pcu::Time(); if (argc != 3) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -143,7 +144,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); printf("---- Creating Mesh Wrapper Object: Done. \n"); // add size distribution based on area @@ -174,7 +175,7 @@ int main(int argc, char** argv) printf("total bin count is %lu \n", totalBinCount); gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneAnalyzeEdgeLengths.cc b/capstone_clis/capStoneAnalyzeEdgeLengths.cc index 31dd09403..df2cb297a 100644 --- a/capstone_clis/capStoneAnalyzeEdgeLengths.cc +++ b/capstone_clis/capStoneAnalyzeEdgeLengths.cc @@ -49,12 +49,13 @@ void edgeLengthStats(apf::Mesh2* m); int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - double initialTime = PCU_Time(); + double initialTime = pcu::Time(); if (argc != 3) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -144,7 +145,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); printf("---- Creating Mesh Wrapper Object: Done. \n"); // add size distribution based on area @@ -153,7 +154,7 @@ int main(int argc, char** argv) edgeLengthStats(mesh); gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneAnisoAdapt.cc b/capstone_clis/capStoneAnisoAdapt.cc index c6d71cff9..da6011294 100644 --- a/capstone_clis/capStoneAnisoAdapt.cc +++ b/capstone_clis/capStoneAnisoAdapt.cc @@ -49,10 +49,11 @@ int main(int argc, char** argv) { /* INIT CALLS */ MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); if (argc < 3) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE: %s \n", argv[0]); printf("Size-fields:\n"); printf("%d, for uniform anisotropic size-field\n", 1); @@ -110,7 +111,7 @@ int main(int argc, char** argv) MG_API_CALL(m, compute_adjacency()); /* CONVERT THE MESH TO APF::MESH2 */ - apf::Mesh2* apfCapMesh = apf::createMesh(m, g); + apf::Mesh2* apfCapMesh = apf::createMesh(m, g, &PCUObj); /* ADD A TEST FIELD TO THE MESH TO DEMONSTRATE SOLUTION TRANSFER */ apf::Field* tf = apf::createFieldOn(apfCapMesh, "test_field", apf::VECTOR); @@ -197,6 +198,6 @@ int main(int argc, char** argv) /* EXIT CALLS */ gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneAnisoAdaptWing.cc b/capstone_clis/capStoneAnisoAdaptWing.cc index 5db6666be..02167ab21 100644 --- a/capstone_clis/capStoneAnisoAdaptWing.cc +++ b/capstone_clis/capStoneAnisoAdaptWing.cc @@ -93,10 +93,11 @@ int main(int argc, char** argv) { /* INIT CALLS */ MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); if (argc < 2) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE: %s \n", argv[0]); } MPI_Finalize(); @@ -149,7 +150,7 @@ int main(int argc, char** argv) MG_API_CALL(m, compute_adjacency()); /* CONVERT THE MESH TO APF::MESH2 */ - apf::Mesh2* apfCapMesh = apf::createMesh(m, g); + apf::Mesh2* apfCapMesh = apf::createMesh(m, g, &PCUObj); /* ADD A TEST FIELD TO THE MESH TO DEMONSTRATE SOLUTION TRANSFER */ apf::Field* tf = apf::createFieldOn(apfCapMesh, "test_field", apf::VECTOR); @@ -223,6 +224,6 @@ int main(int argc, char** argv) /* EXIT CALLS */ gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneAttachSolution.cc b/capstone_clis/capStoneAttachSolution.cc index 28b850a78..e951631ae 100644 --- a/capstone_clis/capStoneAttachSolution.cc +++ b/capstone_clis/capStoneAttachSolution.cc @@ -149,8 +149,8 @@ int gradeSizeModify(apf::Mesh* m, apf::Field* size_iso,double gradingFactor, m->getRemotes(edgAdjVert[idx1],remotes); double newSize = gradingFactor*size[idx2]; int owningPart=m->getOwner(edgAdjVert[idx1]); - PCU_COMM_PACK(owningPart, remotes[owningPart]); - PCU_COMM_PACK(owningPart,newSize); + m->getPCU()->Pack(owningPart, remotes[owningPart]); + m->getPCU()->Pack(owningPart,newSize); } } @@ -249,11 +249,11 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) int needsParallel=1; while(needsParallel) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); needsParallel = serialGradation(m,size_iso,markedEdges,gradingFactor); - PCU_Add_Ints(&needsParallel,1); - PCU_Comm_Send(); + m->getPCU()->Add(&needsParallel,1); + m->getPCU()->Send(); apf::MeshEntity* ent; double receivedSize; @@ -265,10 +265,10 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) apf::Copies remotes; //owning copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { - PCU_COMM_UNPACK(ent); - PCU_COMM_UNPACK(receivedSize); + m->getPCU()->Unpack(ent); + m->getPCU()->Unpack(receivedSize); if(!m->isOwned(ent)){ std::cout<<"THERE WAS AN ERROR"<getPCU()->Begin(); while(!updateRemoteVertices.empty()) { @@ -305,17 +305,17 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) currentSize = apf::getScalar(size_iso,ent,0); for(apf::Copies::iterator iter=remotes.begin(); iter!=remotes.end();++iter) { - PCU_COMM_PACK(iter->first, iter->second); + m->getPCU()->Pack(iter->first, iter->second); } updateRemoteVertices.pop(); } - PCU_Comm_Send(); + m->getPCU()->Send(); //while remote copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { //unpack - PCU_COMM_UNPACK(ent); + m->getPCU()->Unpack(ent); //PCU_COMM_UNPACK(receivedSize); assert(!m->isOwned(ent)); @@ -598,12 +598,13 @@ void isotropicIntersect(apf::Mesh* m, std::queue sizeFieldList, int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - double initialTime = PCU_Time(); + double initialTime = pcu::Time(); if (argc != 9) { - if(0==PCU_Comm_Self()) { + if(0==PCUObj.Self()) { std::cerr << "usage: " << argv[0] << " \n"; std::cerr << "*target field(s) is a bit string to select which field(s) are used for error estimation\n"; @@ -711,7 +712,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); printf("---- Creating Mesh Wrapper Object: Done. \n"); // make the volume mesh (this one is MDS underneath) @@ -735,7 +736,7 @@ int main(int argc, char** argv) printf("number of mesh regions(hexes): %zu\n", volMesh->count(3)); printf("---- Printing Volume/Strand Mesh Stats: Done. \n"); - double constructionTime = PCU_Time(); + double constructionTime = pcu::Time(); std::cout<<"TIMER: Finished converting capstone mesh to volume mesh "<end(itVol); std::cout<<"Finished surface fields from volume\n"; - double getSurfaceTime = PCU_Time(); + double getSurfaceTime = pcu::Time(); std::cout<<"TIMER: Finished computing speed and transferring fields to surface "<debugFolder = "debug"; */ ma::adaptVerbose(in, false); - double adaptTime = PCU_Time(); + double adaptTime = pcu::Time(); std::cout<<"TIMER: adaptMesh "<getRemotes(edgAdjVert[idx1],remotes); double newSize = gradingFactor*size[idx2]; int owningPart=m->getOwner(edgAdjVert[idx1]); - PCU_COMM_PACK(owningPart, remotes[owningPart]); - PCU_COMM_PACK(owningPart,newSize); + m->getPCU()->Pack(owningPart, remotes[owningPart]); + m->getPCU()->Pack(owningPart,newSize); } } @@ -221,11 +221,11 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) int needsParallel=1; while(needsParallel) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); needsParallel = serialGradation(m,size_iso,markedEdges,gradingFactor); - PCU_Add_Ints(&needsParallel,1); - PCU_Comm_Send(); + m->getPCU()->Add(&needsParallel,1); + m->getPCU()->Send(); apf::MeshEntity* ent; double receivedSize; @@ -237,10 +237,10 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) apf::Copies remotes; //owning copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { - PCU_COMM_UNPACK(ent); - PCU_COMM_UNPACK(receivedSize); + m->getPCU()->Unpack(ent); + m->getPCU()->Unpack(receivedSize); if(!m->isOwned(ent)){ std::cout<<"THERE WAS AN ERROR"<getPCU()->Begin(); while(!updateRemoteVertices.empty()) { @@ -277,17 +277,17 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) currentSize = apf::getScalar(size_iso,ent,0); for(apf::Copies::iterator iter=remotes.begin(); iter!=remotes.end();++iter) { - PCU_COMM_PACK(iter->first, iter->second); + m->getPCU()->Pack(iter->first, iter->second); } updateRemoteVertices.pop(); } - PCU_Comm_Send(); + m->getPCU()->Send(); //while remote copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { //unpack - PCU_COMM_UNPACK(ent); + m->getPCU()->Unpack(ent); //PCU_COMM_UNPACK(receivedSize); assert(!m->isOwned(ent)); @@ -493,12 +493,13 @@ void isotropicIntersect(apf::Mesh* m, std::queue sizeFieldList,apf: int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); - double initialTime = PCU_Time(); + double initialTime = pcu::Time(); if (argc != 7) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -597,7 +598,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); printf("---- Creating Mesh Wrapper Object: Done. \n"); // remove unused verts @@ -628,7 +629,7 @@ int main(int argc, char** argv) printf("number of mesh regions(hexes): %zu\n", volMesh->count(3)); printf("---- Printing Volume/Strand Mesh Stats: Done. \n"); - double constructionTime = PCU_Time(); + double constructionTime = pcu::Time(); std::cout<<"TIMER: Finished converting capstone mesh to volume mesh "<end(itVol); std::cout<<"Finished surface fields from volume\n"; - double getSurfaceTime = PCU_Time(); + double getSurfaceTime = pcu::Time(); std::cout<<"TIMER: Finished computing speed and transferring fields to surface "<shouldForceAdaptation = true; ma::adaptVerbose(in); - double adaptTime = PCU_Time(); + double adaptTime = pcu::Time(); std::cout<<"TIMER: adaptMesh "<getRemotes(edgAdjVert[idx1],remotes); double newSize = gradingFactor*size[idx2]; int owningPart=m->getOwner(edgAdjVert[idx1]); - PCU_COMM_PACK(owningPart, remotes[owningPart]); - PCU_COMM_PACK(owningPart,newSize); + m->getPCU()->Pack(owningPart, remotes[owningPart]); + m->getPCU()->Pack(owningPart,newSize); } } @@ -247,11 +247,11 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) int needsParallel=1; while(needsParallel) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); needsParallel = serialGradation(m,size_iso,markedEdges,gradingFactor); - PCU_Add_Ints(&needsParallel,1); - PCU_Comm_Send(); + m->getPCU()->Add(&needsParallel,1); + m->getPCU()->Send(); apf::MeshEntity* ent; double receivedSize; @@ -263,10 +263,10 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) apf::Copies remotes; //owning copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { - PCU_COMM_UNPACK(ent); - PCU_COMM_UNPACK(receivedSize); + m->getPCU()->Unpack(ent); + m->getPCU()->Unpack(receivedSize); if(!m->isOwned(ent)){ std::cout<<"THERE WAS AN ERROR"<getPCU()->Begin(); while(!updateRemoteVertices.empty()) { @@ -303,17 +303,17 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) currentSize = apf::getScalar(size_iso,ent,0); for(apf::Copies::iterator iter=remotes.begin(); iter!=remotes.end();++iter) { - PCU_COMM_PACK(iter->first, iter->second); + m->getPCU()->Pack(iter->first, iter->second); } updateRemoteVertices.pop(); } - PCU_Comm_Send(); + m->getPCU()->Send(); //while remote copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { //unpack - PCU_COMM_UNPACK(ent); + m->getPCU()->Unpack(ent); //PCU_COMM_UNPACK(receivedSize); assert(!m->isOwned(ent)); @@ -520,12 +520,13 @@ void isotropicIntersect(apf::Mesh* m, std::queue sizeFieldList, con int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - double initialTime = PCU_Time(); + double initialTime = pcu::Time(); if (argc != 7) { - if(0==PCU_Comm_Self()) { + if(0==PCUObj.Self()) { std::cerr << "usage: " << argv[0] << " \n"; std::cerr << "*target field(s) is a bit string to select which field(s) are used for error estimation\n"; @@ -556,7 +557,7 @@ int main(int argc, char** argv) // create the mesh object (this one is CapStone underneath) printf("\n---- Creating Mesh Wrapper Object. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); printf("---- Creating Mesh Wrapper Object: Done. \n"); // make the volume mesh (this one is MDS underneath) @@ -581,7 +582,7 @@ int main(int argc, char** argv) printf("number of mesh regions(hexes): %zu\n", volMesh->count(3)); printf("---- Printing Volume/Strand Mesh Stats: Done. \n"); - double constructionTime = PCU_Time(); + double constructionTime = pcu::Time(); std::cout<<"TIMER: Finished converting capstone mesh to volume mesh "<end(itVol); std::cout<<"Finished surface fields from volume\n"; - double getSurfaceTime = PCU_Time(); + double getSurfaceTime = pcu::Time(); std::cout<<"TIMER: Finished computing speed and transferring fields to surface "<debugFolder = "debug"; */ ma::adaptVerbose(in, false); - double adaptTime = PCU_Time(); + double adaptTime = pcu::Time(); std::cout<<"TIMER: adaptMesh "<getRemotes(edgAdjVert[idx1],remotes); double newSize = gradingFactor*size[idx2]; int owningPart=m->getOwner(edgAdjVert[idx1]); - PCU_COMM_PACK(owningPart, remotes[owningPart]); - PCU_COMM_PACK(owningPart,newSize); + m->getPCU()->Pack(owningPart, remotes[owningPart]); + m->getPCU()->Pack(owningPart,newSize); } } @@ -225,11 +225,11 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) int needsParallel=1; while(needsParallel) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); needsParallel = serialGradation(m,size_iso,markedEdges,gradingFactor); - PCU_Add_Ints(&needsParallel,1); - PCU_Comm_Send(); + m->getPCU()->Add(&needsParallel,1); + m->getPCU()->Send(); apf::MeshEntity* ent; double receivedSize; @@ -241,10 +241,10 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) apf::Copies remotes; //owning copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { - PCU_COMM_UNPACK(ent); - PCU_COMM_UNPACK(receivedSize); + m->getPCU()->Unpack(ent); + m->getPCU()->Unpack(receivedSize); if(!m->isOwned(ent)){ std::cout<<"THERE WAS AN ERROR"<getPCU()->Begin(); while(!updateRemoteVertices.empty()) { @@ -281,17 +281,17 @@ int gradeMesh(apf::Mesh* m,apf::Field* size_iso) currentSize = apf::getScalar(size_iso,ent,0); for(apf::Copies::iterator iter=remotes.begin(); iter!=remotes.end();++iter) { - PCU_COMM_PACK(iter->first, iter->second); + m->getPCU()->Pack(iter->first, iter->second); } updateRemoteVertices.pop(); } - PCU_Comm_Send(); + m->getPCU()->Send(); //while remote copies are receiving - while(PCU_Comm_Receive()) + while(m->getPCU()->Receive()) { //unpack - PCU_COMM_UNPACK(ent); + m->getPCU()->Unpack(ent); //PCU_COMM_UNPACK(receivedSize); assert(!m->isOwned(ent)); @@ -497,12 +497,13 @@ void isotropicIntersect(apf::Mesh* m, std::queue sizeFieldList,apf: int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - double initialTime = PCU_Time(); + double initialTime = pcu::Time(); if (argc != 7) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -594,7 +595,7 @@ int main(int argc, char** argv) printf("---- CapStone Mesh Loaded. \n"); - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); //adapt the mesh ma::Input* in; @@ -612,7 +613,7 @@ int main(int argc, char** argv) in->debugFolder = "debug"; ma::adaptVerbose(in, true); - double adaptTime = PCU_Time(); + double adaptTime = pcu::Time(); // write the adapted mesh to vtk @@ -635,7 +636,7 @@ int main(int argc, char** argv) } gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneCheckParametrization.cc b/capstone_clis/capStoneCheckParametrization.cc index 704836d00..3ed3adf03 100644 --- a/capstone_clis/capStoneCheckParametrization.cc +++ b/capstone_clis/capStoneCheckParametrization.cc @@ -28,10 +28,11 @@ void checkParametrization(MeshDatabaseInterface* mdb, GeometryDatabaseInterface* int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); if (argc != 2) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -112,7 +113,7 @@ int main(int argc, char** argv) // check parametrization using capstone apis checkParametrization(m, g); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneCurve.cc b/capstone_clis/capStoneCurve.cc index 8d525da10..147cc007b 100644 --- a/capstone_clis/capStoneCurve.cc +++ b/capstone_clis/capStoneCurve.cc @@ -101,7 +101,8 @@ void writeVtk(CapstoneModule& cs, const std::string& vtkFileName) int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); lion_set_verbosity(1); @@ -160,7 +161,7 @@ int main(int argc, char** argv) MG_API_CALL(m, compute_adjacency()); /* writeVtk(cs, "before.vtk"); */ - apf::Mesh2* apfCapMesh = apf::createMesh(m, g); + apf::Mesh2* apfCapMesh = apf::createMesh(m, g, &PCUObj); apf::Field* tf = apf::createFieldOn(apfCapMesh, "test_field", apf::VECTOR); apf::MeshEntity* ent; @@ -195,6 +196,6 @@ int main(int argc, char** argv) /* writeVtk(cs, "after.vtk"); */ gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/capstone_clis/capStoneGeomTest.cc b/capstone_clis/capStoneGeomTest.cc index dd2756134..9d750523b 100644 --- a/capstone_clis/capStoneGeomTest.cc +++ b/capstone_clis/capStoneGeomTest.cc @@ -39,15 +39,16 @@ char const* const typeName[4] = "region"}; void printInfo(gmi_model* model, int dim); -void visualizeFace(gmi_model* model, gmi_ent* entity, int n, int m, const char* fileName); -void visualizeEdge(gmi_model* model, gmi_ent* entity, int n, const char* fileName); -void visualizeEdges(gmi_model* model, int n, const char* fileName); +void visualizeFace(gmi_model* model, gmi_ent* entity, int n, int m, const char* fileName, pcu::PCU *PCUObj); +void visualizeEdge(gmi_model* model, gmi_ent* entity, int n, const char* fileName, pcu::PCU *PCUObj); +void visualizeEdges(gmi_model* model, int n, const char* fileName, pcu::PCU *PCUObj); int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + auto PCUObj = std::unique_ptr(new pcu::PCU(MPI_COMM_WORLD)); gmi_register_mesh(); gmi_register_null(); @@ -115,7 +116,7 @@ int main(int argc, char** argv) gmi_register_cap(); - apf::Mesh2* mesh0 = apf::createMesh(m,g); + apf::Mesh2* mesh0 = apf::createMesh(m,g,&PCUObj); apf::writeVtkFiles("mesh_no_param", mesh0); gmi_model* model = gmi_import_cap(g); @@ -136,7 +137,7 @@ int main(int argc, char** argv) while( (ge = gmi_next(model, gi)) ){ std::stringstream name_str; name_str << "face_" << gmi_tag(model, ge) << "_mesh"; - visualizeFace(model, ge, 100, 100, name_str.str().c_str()); + visualizeFace(model, ge, 100, 100, name_str.str().c_str(), &PCUObj); } gmi_end(model, gi); @@ -144,7 +145,7 @@ int main(int argc, char** argv) printf("creating mesh with param field\n"); - apf::Mesh2* mesh = apf::createMesh(m,g); + apf::Mesh2* mesh = apf::createMesh(m,g,&PCUObj); apf::Field* pf = apf::createFieldOn(mesh, "param_field", apf::VECTOR); apf::Field* idf = apf::createFieldOn(mesh, "id", apf::SCALAR); apf::MeshEntity* e; @@ -162,7 +163,7 @@ int main(int argc, char** argv) gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -236,7 +237,7 @@ void printInfo(gmi_model* model, int dim) printf("-------------------------------------------\n"); } -void visualizeFace(gmi_model* model, gmi_ent* entity, int n, int m, const char* fileName) +void visualizeFace(gmi_model* model, gmi_ent* entity, int n, int m, const char* fileName, pcu::PCU *PCUObj) { // assert type is 2 // get the range @@ -270,7 +271,7 @@ void visualizeFace(gmi_model* model, gmi_ent* entity, int n, int m, const char* // make the vertexes and set the coordinates using the array std::vector vs; - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false, &PCUObj); for (size_t i = 0; i < ps.size(); i++) { ma::Entity* vert = mesh->createVert(0); mesh->setPoint(vert, 0, ps[i]); @@ -341,7 +342,7 @@ void visualizeFace(gmi_model* model, gmi_ent* entity, int n, int m, const char* } -void visualizeEdge(gmi_model* model, gmi_ent* entity, int n, const char* fileName) +void visualizeEdge(gmi_model* model, gmi_ent* entity, int n, const char* fileName, pcu::PCU *PCUObj) { // assert type is 1 // get the range @@ -365,7 +366,7 @@ void visualizeEdge(gmi_model* model, gmi_ent* entity, int n, const char* fileNam // make the vertexes and set the coordinates using the array std::vector vs; - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false, pcu::PCU *PCUObj); for (size_t i = 0; i < ps.size(); i++) { ma::Entity* vert = mesh->createVert(0); mesh->setPoint(vert, 0, ps[i]); @@ -404,9 +405,9 @@ void visualizeEdge(gmi_model* model, gmi_ent* entity, int n, const char* fileNam apf::destroyMesh(mesh); } -void visualizeEdges(gmi_model* model, int n, const char* fileName) +void visualizeEdges(gmi_model* model, int n, const char* fileName, pcu::PCU *PCUObj) { - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false, PCUObj); gmi_ent* entity; gmi_iter* gi = gmi_begin(model, 1); while( (entity = gmi_next(model, gi)) ){ diff --git a/capstone_clis/capStoneIsoAdaptB737.cc b/capstone_clis/capStoneIsoAdaptB737.cc index e2cf07f00..b52d2b4d3 100644 --- a/capstone_clis/capStoneIsoAdaptB737.cc +++ b/capstone_clis/capStoneIsoAdaptB737.cc @@ -217,10 +217,11 @@ int main(int argc, char** argv) { /* INIT CALLS */ MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); if (argc < 2) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE: %s \n", argv[0]); } MPI_Finalize(); @@ -273,7 +274,7 @@ int main(int argc, char** argv) MG_API_CALL(m, compute_adjacency()); /* CONVERT THE MESH TO APF::MESH2 */ - apf::Mesh2* apfCapMesh = apf::createMesh(m, g); + apf::Mesh2* apfCapMesh = apf::createMesh(m, g, &PCUObj); /* ADD A TEST FIELD TO THE MESH TO DEMONSTRATE SOLUTION TRANSFER */ apf::Field* tf = apf::createFieldOn(apfCapMesh, "test_field", apf::VECTOR); @@ -374,6 +375,6 @@ int main(int argc, char** argv) /* EXIT CALLS */ gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/crv/crvAdapt.cc b/crv/crvAdapt.cc index 12d58abb1..6d2ab52b4 100644 --- a/crv/crvAdapt.cc +++ b/crv/crvAdapt.cc @@ -15,7 +15,6 @@ #include #include #include -#include #include namespace crv { @@ -73,15 +72,15 @@ void splitEdges(ma::Adapt* a) static void refine(ma::Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); --(a->refinesLeft); long count = ma::markEdgesToSplit(a); if ( ! count) { return; } splitEdges(a); - double t1 = PCU_Time(); - ma::print("split %li edges in %f seconds",count,t1-t0); + double t1 = pcu::Time(); + ma::print(a->mesh->getPCU(), "split %li edges in %f seconds", count, t1-t0); } int getValidityTag(ma::Mesh* m, ma::Entity* e, @@ -132,7 +131,7 @@ int markInvalidEntities(Adapt* a) } m->end(it); delete qual; - return PCU_Add_Int(count); + return m->getPCU()->Add(count); } int getTag(Adapt* a, ma::Entity* e) @@ -199,8 +198,8 @@ void adapt(ma::Input* in) fail("mesh must be bezier to adapt\n"); in->shapeHandler = crv::getShapeHandler; - ma::print("Curved Adaptation Version 2.0 !"); - double t0 = PCU_Time(); + ma::print(in->mesh->getPCU(), "Curved Adaptation Version 2.0 !"); + double t0 = pcu::Time(); ma::validateInput(in); Adapt* a = new Adapt(in); ma::preBalance(a); @@ -209,7 +208,7 @@ void adapt(ma::Input* in) for (int i=0; i < in->maximumIterations; ++i) { - ma::print("iteration %d",i); + ma::print(a->mesh->getPCU(), "iteration %d", i); ma::coarsen(a); ma::midBalance(a); crv::refine(a); @@ -228,8 +227,8 @@ void adapt(ma::Input* in) cleanupLayer(a); ma::printQuality(a); ma::postBalance(a); - double t1 = PCU_Time(); - ma::print("mesh adapted in %f seconds",t1-t0); + double t1 = pcu::Time(); + ma::print(a->mesh->getPCU(), "mesh adapted in %f seconds", t1-t0); apf::printStats(a->mesh); crv::clearTags(a); delete a; diff --git a/crv/crvShape.cc b/crv/crvShape.cc index 33f421292..6bfc91768 100644 --- a/crv/crvShape.cc +++ b/crv/crvShape.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include #include "crv.h" #include "crvAdapt.h" @@ -635,7 +634,7 @@ static int markEdgesOppLargeAnglesTri(Adapt* a) } m->end(it); } while(count > prev_count); - return PCU_Add_Long(count); + return m->getPCU()->Add(count); } static int markEdgesOppLargeAnglesTet(Adapt* a) @@ -663,7 +662,7 @@ static int markEdgesOppLargeAnglesTet(Adapt* a) } m->end(it); } while(count > prev_count); - return PCU_Add_Long(count); + return m->getPCU()->Add(count); } /* The whole idea is to do the quality check once, @@ -701,12 +700,12 @@ static int markEdgesToFix(Adapt* a, int flag) } m->end(it); - return PCU_Add_Long(count); + return m->getPCU()->Add(count); } int fixLargeBoundaryAngles(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); int count = markEdgesOppLargeAnglesTet(a); count += markEdgesOppLargeAnglesTri(a); @@ -714,15 +713,15 @@ int fixLargeBoundaryAngles(Adapt* a) return 0; } splitEdges(a); - double t1 = PCU_Time(); - ma::print("split %d boundary edges with " - "large angles in %f seconds",count,t1-t0); + double t1 = pcu::Time(); + ma::print(a->mesh->getPCU(), "split %d boundary edges with " + "large angles in %f seconds", count, t1-t0); return 0; } static void collapseInvalidEdges(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); ma::Mesh* m = a->mesh; int maxDimension = m->getDimension(); PCU_ALWAYS_ASSERT(checkFlagConsistency(a,1,ma::COLLAPSE)); @@ -733,30 +732,30 @@ static void collapseInvalidEdges(Adapt* a) findIndependentSet(a); successCount += ma::collapseAllEdges(a, modelDimension); } - successCount = PCU_Add_Long(successCount); - double t1 = PCU_Time(); - ma::print("Collapsed %d bad edges " - "in %f seconds",successCount, t1-t0); + successCount = m->getPCU()->Add(successCount); + double t1 = pcu::Time(); + ma::print(m->getPCU(), "Collapsed %d bad edges " + "in %f seconds", successCount, t1-t0); } static void swapInvalidEdges(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); EdgeSwapper es(a); ma::applyOperator(a,&es); - double t1 = PCU_Time(); - ma::print("Swapped %d bad edges " - "in %f seconds",es.ns, t1-t0); + double t1 = pcu::Time(); + ma::print(a->mesh->getPCU(), "Swapped %d bad edges " + "in %f seconds", es.ns, t1-t0); } static void repositionInvalidEdges(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); EdgeReshaper es(a); ma::applyOperator(a,&es); - double t1 = PCU_Time(); - ma::print("Repositioned %d bad edges " - "in %f seconds",es.nr, t1-t0); + double t1 = pcu::Time(); + ma::print(a->mesh->getPCU(), "Repositioned %d bad edges " + "in %f seconds", es.nr, t1-t0); } int fixInvalidEdges(Adapt* a) @@ -827,7 +826,7 @@ void fixCrvElementShapes(Adapt* a) if ( ! a->input->shouldFixShape) return; a->input->shouldForceAdaptation = true; - double t0 = PCU_Time(); + double t0 = pcu::Time(); int count = markCrvBadQuality(a); int originalCount = count; int prev_count; @@ -850,8 +849,8 @@ void fixCrvElementShapes(Adapt* a) count = markCrvBadQuality(a); ++i; } while(count < prev_count && i < 6); // the second conditions is to make sure this does not take long - double t1 = PCU_Time(); - ma::print("bad shapes down from %d to %d in %f seconds", + double t1 = pcu::Time(); + ma::print(a->mesh->getPCU(), "bad shapes down from %d to %d in %f seconds", originalCount,count,t1-t0); a->input->shouldForceAdaptation = false; } diff --git a/crv/crvShapeFixer.cc b/crv/crvShapeFixer.cc index b05fac91f..a063ca2ec 100644 --- a/crv/crvShapeFixer.cc +++ b/crv/crvShapeFixer.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "crvShapeFixer.h" #include "crvTables.h" #include "maTables.h" diff --git a/crv/crvShapeFixer.h b/crv/crvShapeFixer.h index 4757675a7..1233e4eae 100644 --- a/crv/crvShapeFixer.h +++ b/crv/crvShapeFixer.h @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "crvAdapt.h" #include "maOperator.h" #include "maEdgeSwap.h" diff --git a/crv/crvVtk.cc b/crv/crvVtk.cc index dd72bafac..73dcacf30 100644 --- a/crv/crvVtk.cc +++ b/crv/crvVtk.cc @@ -6,7 +6,6 @@ */ #include "crv.h" -#include "PCU.h" #include "apfDynamicVector.h" #include "apfFieldData.h" #include "apfMDS.h" @@ -737,7 +736,7 @@ static void writePvtuFile(const char* prefix, const char* suffix, } writePPointData(file,m); file << "\n"; - for (int i=0; i < PCU_Comm_Peers(); ++i) + for (int i=0; i < m->getPCU()->Peers(); ++i) { std::stringstream ssPart; ssPart << "vtu/" @@ -764,13 +763,13 @@ static void writePointData(std::ostream& file, apf::Mesh* m, void writeInterpolationPointVtuFiles(apf::Mesh* m, const char* prefix) { - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) writePvtuFile(prefix,"_interPts",m,apf::Mesh::VERTEX); - PCU_Barrier(); + m->getPCU()->Barrier(); std::stringstream ss; - ss << prefix << PCU_Comm_Self() << "_interPts" + ss << prefix << m->getPCU()->Self() << "_interPts" << "_" << m->getShape()->getOrder() << ".vtu"; @@ -831,18 +830,18 @@ void writeInterpolationPointVtuFiles(apf::Mesh* m, const char* prefix) file << buf.rdbuf(); } - PCU_Barrier(); + m->getPCU()->Barrier(); } void writeControlPointVtuFiles(apf::Mesh* m, const char* prefix) { - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) writePvtuFile(prefix,getSuffix(apf::Mesh::VERTEX),m,apf::Mesh::VERTEX); - PCU_Barrier(); + m->getPCU()->Barrier(); std::stringstream ss; - ss << prefix << PCU_Comm_Self() + ss << prefix << m->getPCU()->Self() << getSuffix(apf::Mesh::VERTEX) << "_" << m->getShape()->getOrder() << ".vtu"; @@ -901,7 +900,7 @@ void writeControlPointVtuFiles(apf::Mesh* m, const char* prefix) file << buf.rdbuf(); } - PCU_Barrier(); + m->getPCU()->Barrier(); } static void safe_mkdir(const char* path) @@ -949,7 +948,7 @@ static void makeDirectories(const char* prefix, int type, int n) void writeCurvedWireFrame(apf::Mesh* m, int n, const char* prefix) { - apf::Mesh2* wireMesh = apf::makeEmptyMdsMesh(NULL, 1, false); + apf::Mesh2* wireMesh = apf::makeEmptyMdsMesh(NULL, 1, false, m->getPCU()); apf::Field* f = m->getCoordinateField(); apf::MeshEntity* ent; apf::MeshIterator* it; @@ -1001,17 +1000,17 @@ void writeCurvedWireFrame(apf::Mesh* m, int n, const char* prefix) void writeCurvedVtuFiles(apf::Mesh* m, int type, int n, const char* prefix) { - double t0 = PCU_Time(); - if (!PCU_Comm_Self()) { + double t0 = pcu::Time(); + if (!m->getPCU()->Self()) { makeDirectories(prefix, type, n); writePvtuFile(getPvtuDirectoryStr(prefix, type, n).c_str(),"",m,type); } - PCU_Barrier(); + m->getPCU()->Barrier(); std::stringstream ss; ss << getVtuDirectoryStr(prefix, type, n) << "/order_" << m->getShape()->getOrder() << "_" - << PCU_Comm_Self() + << m->getPCU()->Self() << ".vtu"; std::string fileName = ss.str(); std::stringstream buf; @@ -1055,9 +1054,9 @@ void writeCurvedVtuFiles(apf::Mesh* m, int type, int n, const char* prefix) file << buf.rdbuf(); } - PCU_Barrier(); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + m->getPCU()->Barrier(); + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) lion_oprint(1,"%s vtk files %s written in %f seconds\n", apf::Mesh::typeName[type],getPvtuDirectoryStr(prefix, type, n).c_str(),t1 - t0); } diff --git a/dsp/dspGraphDistance.cc b/dsp/dspGraphDistance.cc index 4b2ec8eb7..03dd79e56 100644 --- a/dsp/dspGraphDistance.cc +++ b/dsp/dspGraphDistance.cc @@ -1,5 +1,4 @@ #include "dspGraphDistance.h" -#include #include namespace dsp { @@ -39,7 +38,7 @@ apf::Numbering* getGraphDistance(apf::Mesh* m, Boundary& seed, } } m->end(it); - for (int layer = 0; PCU_Or(!empty(vs, first)); ++layer) { + for (int layer = 0; m->getPCU()->Or(!empty(vs, first)); ++layer) { size_t layer_end = vs.size(); while (first < layer_end) { v = pop(vs, first); @@ -56,7 +55,7 @@ apf::Numbering* getGraphDistance(apf::Mesh* m, Boundary& seed, } } } - PCU_Comm_Begin(); + m->getPCU()->Begin(); apf::MeshEntity* sv; for (size_t i = first; i < vs.size(); ++i) { sv = vs[i]; @@ -67,11 +66,11 @@ apf::Numbering* getGraphDistance(apf::Mesh* m, Boundary& seed, apf::Copies remotes; m->getRemotes(sv, remotes); APF_ITERATE(apf::Copies, remotes, rit) - PCU_COMM_PACK(rit->first, rit->second); + m->getPCU()->Pack(rit->first, rit->second); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { - PCU_COMM_UNPACK(sv); + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { + m->getPCU()->Unpack(sv); if (!apf::isNumbered(dst, sv, 0, 0)) { apf::number(dst, sv, 0, 0, layer + 1); push(vs, sv); diff --git a/gmi_cap/gmi_cap.cc b/gmi_cap/gmi_cap.cc index 851c331ea..12c393cd3 100644 --- a/gmi_cap/gmi_cap.cc +++ b/gmi_cap/gmi_cap.cc @@ -7,7 +7,6 @@ BSD license as described in the LICENSE file in the top-level directory. *******************************************************************************/ -#include #include "gmi_cap.h" #include #include diff --git a/gmi_sim/gmi_sim.cc b/gmi_sim/gmi_sim.cc index b89375746..e79b25d6b 100644 --- a/gmi_sim/gmi_sim.cc +++ b/gmi_sim/gmi_sim.cc @@ -7,7 +7,6 @@ BSD license as described in the LICENSE file in the top-level directory. *******************************************************************************/ -#include #include "gmi_sim.h" #include #include diff --git a/ma/ma.cc b/ma/ma.cc index 64d692e25..24c5100ce 100644 --- a/ma/ma.cc +++ b/ma/ma.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "ma.h" #include "maAdapt.h" #include "maCoarsen.h" @@ -23,14 +22,14 @@ namespace ma { void adapt(Input* in) { - print("version 2.0 !"); - double t0 = PCU_Time(); + double t0 = pcu::Time(); + print(in->mesh->getPCU(), "version 2.0 !"); validateInput(in); Adapt* a = new Adapt(in); preBalance(a); for (int i = 0; i < in->maximumIterations; ++i) { - print("iteration %d",i); + print(a->mesh->getPCU(), "iteration %d", i); coarsen(a); coarsenLayer(a); midBalance(a); @@ -51,8 +50,8 @@ void adapt(Input* in) if (in->ownsSolutionTransfer) delete in->solutionTransfer; delete in; - double t1 = PCU_Time(); - print("mesh adapted in %f seconds",t1-t0); + double t1 = pcu::Time(); + print(m->getPCU(), "mesh adapted in %f seconds", t1-t0); apf::printStats(m); } @@ -63,14 +62,14 @@ void adapt(const Input* in) void adaptVerbose(Input* in, bool verbose) { - print("version 2.0 - dev !"); - double t0 = PCU_Time(); + double t0 = pcu::Time(); + print(in->mesh->getPCU(), "version 2.0 - dev !"); validateInput(in); Adapt* a = new Adapt(in); preBalance(a); for (int i = 0; i < in->maximumIterations; ++i) { - print("iteration %d",i); + print(a->mesh->getPCU(), "iteration %d", i); coarsen(a); if (verbose && in->shouldCoarsen) ma_dbg::dumpMeshWithQualities(a,i,"after_coarsen"); @@ -100,14 +99,14 @@ void adaptVerbose(Input* in, bool verbose) */ int count = 0; double lMax = ma::getMaximumEdgeLength(a->mesh, a->sizeField); - print("Maximum (metric) edge length in the mesh is %f", lMax); + print(a->mesh->getPCU(), "Maximum (metric) edge length in the mesh is %f", lMax); while (lMax > 1.5) { - print("%dth additional refine-snap call", count); + print(a->mesh->getPCU(), "%dth additional refine-snap call", count); refine(a); snap(a); lMax = ma::getMaximumEdgeLength(a->mesh, a->sizeField); count++; - print("Maximum (metric) edge length in the mesh is %f", lMax); + print(a->mesh->getPCU(), "Maximum (metric) edge length in the mesh is %f", lMax); if (count > 5) break; } if (verbose) @@ -125,8 +124,8 @@ void adaptVerbose(Input* in, bool verbose) if (in->ownsSolutionTransfer) delete in->solutionTransfer; delete in; - double t1 = PCU_Time(); - print("mesh adapted in %f seconds",t1-t0); + double t1 = pcu::Time(); + print(m->getPCU(), "mesh adapted in %f seconds", t1-t0); apf::printStats(m); } diff --git a/ma/maAdapt.cc b/ma/maAdapt.cc index 606e095d5..bff23cf90 100644 --- a/ma/maAdapt.cc +++ b/ma/maAdapt.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include #include "maAdapt.h" #include "maInput.h" @@ -111,7 +110,7 @@ void setFlagMatched(Adapt* a, Entity* e, int flag) apf::Matches matches; a->mesh->getMatches(e, matches); APF_ITERATE(apf::Matches, matches, it) { - PCU_ALWAYS_ASSERT(it->peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(it->peer == a->mesh->getPCU()->Self()); setFlag(a, it->entity, flag); } } @@ -131,7 +130,7 @@ void clearFlagMatched(Adapt* a, Entity* e, int flag) apf::Matches matches; a->mesh->getMatches(e, matches); APF_ITERATE(apf::Matches, matches, it) { - PCU_ALWAYS_ASSERT(it->peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(it->peer == a->mesh->getPCU()->Self()); clearFlag(a, it->entity, flag); } } @@ -228,7 +227,7 @@ bool checkFlagConsistency(Adapt* a, int dimension, int flag) { Mesh* m = a->mesh; apf::Sharing* sh = apf::getSharing(m); - PCU_Comm_Begin(); + m->getPCU()->Begin(); Entity* e; Iterator* it = m->begin(dimension); while ((e = m->iterate(it))) { @@ -238,17 +237,17 @@ bool checkFlagConsistency(Adapt* a, int dimension, int flag) continue; bool value = getFlag(a, e, flag); APF_ITERATE(apf::CopyArray, others, rit) { - PCU_COMM_PACK(rit->peer, rit->entity); - PCU_COMM_PACK(rit->peer, value); + m->getPCU()->Pack(rit->peer, rit->entity); + m->getPCU()->Pack(rit->peer, value); } } m->end(it); - PCU_Comm_Send(); + m->getPCU()->Send(); bool ok = true; - while (PCU_Comm_Receive()) { - PCU_COMM_UNPACK(e); + while (m->getPCU()->Receive()) { + m->getPCU()->Unpack(e); bool value; - PCU_COMM_UNPACK(value); + m->getPCU()->Unpack(value); if(value != getFlag(a,e,flag)) ok = false; } @@ -321,7 +320,7 @@ long markEntities( setFlag(a,e,setFalseFlag); } m->end(it); - return PCU_Add_Long(count); + return m->getPCU()->Add(count); } void NewEntities::reset() @@ -471,9 +470,9 @@ void clearBuildCallback(Adapt* a) a->buildCallback = 0; } -void print(const char* format, ...) +void print(pcu::PCU *PCUObj, const char* format, ...) { - if (PCU_Comm_Self()) + if (PCUObj->Self()) return; lion_oprint(1,"\nMeshAdapt: "); va_list ap; @@ -500,7 +499,7 @@ void syncFlag(Adapt* a, int dimension, int flag) { Mesh* m = a->mesh; apf::Sharing* sh = apf::getSharing(m); - PCU_Comm_Begin(); + m->getPCU()->Begin(); Entity* e; Iterator* it = m->begin(dimension); while ((e = m->iterate(it))) { @@ -509,12 +508,12 @@ void syncFlag(Adapt* a, int dimension, int flag) apf::CopyArray others; sh->getCopies(e, others); APF_ITERATE(apf::CopyArray, others, rit) - PCU_COMM_PACK(rit->peer, rit->entity); + m->getPCU()->Pack(rit->peer, rit->entity); } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { - PCU_COMM_UNPACK(e); + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { + m->getPCU()->Unpack(e); setFlag(a,e,flag); } delete sh; diff --git a/ma/maAdapt.h b/ma/maAdapt.h index 81a943e0a..0b7a0324d 100644 --- a/ma/maAdapt.h +++ b/ma/maAdapt.h @@ -164,7 +164,7 @@ Entity* rebuildElement( void setBuildCallback(Adapt* a, apf::BuildCallback* cb); void clearBuildCallback(Adapt* a); -void print(const char* format, ...) __attribute__((format(printf,1,2))); +void print(pcu::PCU *PCUObj, const char* format, ...) __attribute__((format(printf,2,3))); void setFlagOnClosure(Adapt* a, Entity* e, int flag); void syncFlag(Adapt* a, int dimension, int flag); diff --git a/ma/maBalance.cc b/ma/maBalance.cc index edeefa96c..d402fe781 100644 --- a/ma/maBalance.cc +++ b/ma/maBalance.cc @@ -1,4 +1,3 @@ -#include #include "maBalance.h" #include "maAdapt.h" #include @@ -120,7 +119,7 @@ void printEntityImbalance(Mesh* m) double imbalance[4]; Parma_GetEntImbalance(m,&imbalance); double p = (imbalance[m->getDimension()]-1)*100; - print("element imbalance %.0f%% of average",p); + print(m->getPCU(), "element imbalance %.0f%% of average", p); } double estimateWeightedImbalance(Adapt* a) @@ -135,7 +134,7 @@ double estimateWeightedImbalance(Adapt* a) void preBalance(Adapt* a) { - if (PCU_Comm_Peers()==1) + if (a->mesh->getPCU()->Peers()==1) return; Input* in = a->input; // First take care of user overrides. That is, if any of the three options @@ -164,7 +163,7 @@ void preBalance(Adapt* a) #ifdef PUMI_HAS_ZOLTAN // The parmetis multi-level graph partitioner memory usage grows // significantly with process count beyond 16K processes - if (PCU_Comm_Peers() < MAX_ZOLTAN_GRAPH_RANKS) { + if (a->mesh->getPCU()->Peers() < MAX_ZOLTAN_GRAPH_RANKS) { runZoltan(a); return; } @@ -181,7 +180,7 @@ void preBalance(Adapt* a) void midBalance(Adapt* a) { - if (PCU_Comm_Peers()==1) + if (a->mesh->getPCU()->Peers()==1) return; Input* in = a->input; // First take care of user overrides. That is, if any of the three options @@ -204,7 +203,7 @@ void midBalance(Adapt* a) #ifdef PUMI_HAS_ZOLTAN // The parmetis multi-level graph partitioner memory usage grows // significantly with process count beyond 16K processes - if (PCU_Comm_Peers() < MAX_ZOLTAN_GRAPH_RANKS) { + if (a->mesh->getPCU()->Peers() < MAX_ZOLTAN_GRAPH_RANKS) { runZoltan(a); return; } @@ -221,7 +220,7 @@ void midBalance(Adapt* a) void postBalance(Adapt* a) { - if (PCU_Comm_Peers()==1) + if (a->mesh->getPCU()->Peers()==1) return; Input* in = a->input; // First take care of user overrides. That is, if any of the three options @@ -252,7 +251,7 @@ void postBalance(Adapt* a) #ifdef PUMI_HAS_ZOLTAN // The parmetis multi-level graph partitioner memory usage grows // significantly with process count beyond 16K processes - if (PCU_Comm_Peers() < MAX_ZOLTAN_GRAPH_RANKS) { + if (a->mesh->getPCU()->Peers() < MAX_ZOLTAN_GRAPH_RANKS) { runZoltan(a); printEntityImbalance(a->mesh); return; diff --git a/ma/maCoarsen.cc b/ma/maCoarsen.cc index 26379178c..35089e344 100644 --- a/ma/maCoarsen.cc +++ b/ma/maCoarsen.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "maCoarsen.h" #include "maAdapt.h" #include "maCollapse.h" @@ -230,7 +229,7 @@ bool coarsen(Adapt* a) { if (!a->input->shouldCoarsen) return false; - double t0 = PCU_Time(); + double t0 = pcu::Time(); --(a->coarsensLeft); long count = markEdgesToCollapse(a); if ( ! count) @@ -248,9 +247,9 @@ bool coarsen(Adapt* a) else successCount += collapseAllEdges(a, modelDimension); } - successCount = PCU_Add_Long(successCount); - double t1 = PCU_Time(); - print("coarsened %li edges in %f seconds",successCount,t1-t0); + successCount = m->getPCU()->Add(successCount); + double t1 = pcu::Time(); + print(m->getPCU(), "coarsened %li edges in %f seconds", successCount,t1-t0); return true; } diff --git a/ma/maCrawler.cc b/ma/maCrawler.cc index d4c0f6b01..8f97407d4 100644 --- a/ma/maCrawler.cc +++ b/ma/maCrawler.cc @@ -1,4 +1,3 @@ -#include #include "maCrawler.h" #include "maAdapt.h" #include "maLayer.h" @@ -9,24 +8,24 @@ namespace ma { void syncLayer(Crawler* c, Crawler::Layer& layer) { Mesh* m = c->mesh; - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (size_t i = 0; i < layer.size(); ++i) { Entity* e = layer[i]; if (m->isShared(e)) { apf::Copies remotes; m->getRemotes(e,remotes); APF_ITERATE(apf::Copies,remotes,it) { - PCU_COMM_PACK(it->first,it->second); + m->getPCU()->Pack(it->first,it->second); c->send(e, it->first); } } } - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { - int from = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) { + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { + int from = m->getPCU()->Sender(); + while ( ! m->getPCU()->Unpacked()) { Entity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); if (c->recv(e, from)) layer.push_back(e); } @@ -49,7 +48,7 @@ void crawlLayers(Crawler* c) { Crawler::Layer layer; c->begin(layer); - while (PCU_Or( ! layer.empty())) { + while (c->mesh->getPCU()->Or( ! layer.empty())) { crawlLayer(c, layer); syncLayer(c, layer); } @@ -157,12 +156,12 @@ struct LayerNumberer : public Crawler void send(Entity* v, int to) { int n = t.getNumber(v); - PCU_COMM_PACK(to, n); + m->getPCU()->Pack(to, n); } bool recv(Entity* v, int) { int n; - PCU_COMM_UNPACK(n); + m->getPCU()->Unpack(n); if (t.hasNumber(v)) return false; t.setNumber(v, n); diff --git a/ma/maDBG.cc b/ma/maDBG.cc index 69f08c14c..2460ec809 100644 --- a/ma/maDBG.cc +++ b/ma/maDBG.cc @@ -13,7 +13,6 @@ #include "maRefine.h" #include #include -#include #include #include #include @@ -224,7 +223,7 @@ void createCavityMesh(ma::Adapt* a, ma::Mesh* m = a->mesh; gmi_register_null(); - ma::Mesh* cavityMesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 3, false); + ma::Mesh* cavityMesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 3, false, m->getPCU()); size_t n = tets.getSize(); for (size_t i = 0; i < n; ++i) { @@ -365,7 +364,7 @@ void visualizeSizeField( const char* outputPrefix) { // create the size-field visualization mesh - apf::Mesh2* msf = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false); + apf::Mesh2* msf = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false, m->getPCU()); apf::FieldShape* fs = apf::getShape(sizes); int dim = m->getDimension(); diff --git a/ma/maExtrude.cc b/ma/maExtrude.cc index 2fb42c897..13b4088b5 100644 --- a/ma/maExtrude.cc +++ b/ma/maExtrude.cc @@ -1,7 +1,6 @@ #include "maExtrude.h" #include "maCrawler.h" #include -#include #include #include @@ -293,7 +292,7 @@ class DebugBuildCallback : public apf::BuildCallback { void stitchVerts(Mesh* m, Crawler::Layer const& prev_verts, Crawler::Layer const& next_verts, Tag* indices) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); PCU_ALWAYS_ASSERT(prev_verts.size() == next_verts.size()); for (size_t i = 0; i < prev_verts.size(); ++i) { Entity* prev_vert = prev_verts[i]; @@ -303,17 +302,17 @@ void stitchVerts(Mesh* m, Crawler::Layer const& prev_verts, APF_ITERATE(Remotes, remotes, rit) { int remote_part = rit->first; Entity* remote_prev_vert = rit->second; - PCU_COMM_PACK(remote_part, remote_prev_vert); - PCU_COMM_PACK(remote_part, next_vert); + m->getPCU()->Pack(remote_part, remote_prev_vert); + m->getPCU()->Pack(remote_part, next_vert); } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { - int remote_part = PCU_Comm_Sender(); + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { + int remote_part = m->getPCU()->Sender(); Entity* prev_vert; Entity* remote_next_vert; - PCU_COMM_UNPACK(prev_vert); - PCU_COMM_UNPACK(remote_next_vert); + m->getPCU()->Unpack(prev_vert); + m->getPCU()->Unpack(remote_next_vert); int prev_idx; m->getIntTag(prev_vert, indices, &prev_idx); Entity* next_vert = next_verts.at(prev_idx); diff --git a/ma/maInput.cc b/ma/maInput.cc index 9b1c9d25f..fb17deca7 100644 --- a/ma/maInput.cc +++ b/ma/maInput.cc @@ -12,7 +12,6 @@ #include #include #include -#include #include #include @@ -66,9 +65,9 @@ void setDefaultValues(Input* in) in->debugFolder = nullptr; } -void rejectInput(const char* str) +void rejectInput(const char* str, pcu::PCU *PCUObj) { - if (PCU_Comm_Self() != 0) + if (PCUObj->Self() != 0) return; lion_eprint(1,"MeshAdapt input error:\n"); lion_eprint(1,"%s\n",str); @@ -90,62 +89,62 @@ static bool moreThanOneOptionIsTrue(bool op1, bool op2, bool op3) void validateInput(Input* in) { if ( ! in->sizeField) - rejectInput("no size field"); + rejectInput("no size field", in->mesh->getPCU()); if ( ! in->solutionTransfer) - rejectInput("no solution transfer object"); + rejectInput("no solution transfer object", in->mesh->getPCU()); if (in->maximumIterations < 0) - rejectInput("negative maximum iteration count"); + rejectInput("negative maximum iteration count", in->mesh->getPCU()); if (in->maximumIterations > 10) - rejectInput("unusually high maximum iteration count"); + rejectInput("unusually high maximum iteration count", in->mesh->getPCU()); if (in->shouldSnap &&( ! in->mesh->canSnap())) rejectInput("user requested snapping " - "but the geometric model does not support it"); + "but the geometric model does not support it", in->mesh->getPCU()); if (in->shouldTransferParametric &&( ! in->mesh->canSnap())) rejectInput("user requested parametric coordinate transfer " - "but the geometric model does not support it"); + "but the geometric model does not support it", in->mesh->getPCU()); if (in->shouldTransferToClosestPoint &&( ! in->mesh->canSnap())) rejectInput("user requested transfer to closest point on model" - "but the geometric model does not support it"); + "but the geometric model does not support it", in->mesh->getPCU()); if (in->shouldSnap && ( ! (in->shouldTransferParametric || in->shouldTransferToClosestPoint))) - rejectInput("snapping requires parametric coordinate transfer or transfer to closest point"); + rejectInput("snapping requires parametric coordinate transfer or transfer to closest point", in->mesh->getPCU()); if ((in->mesh->hasMatching()) &&( ! in->shouldHandleMatching)) - rejectInput("the mesh has matching entities but matched support is off"); + rejectInput("the mesh has matching entities but matched support is off", in->mesh->getPCU()); if (in->shouldHandleMatching && in->shouldFixShape) rejectInput("user requested matched mesh handling and shape correction " - "but shape correction does not support matching yet"); + "but shape correction does not support matching yet", in->mesh->getPCU()); if (in->goodQuality < 0.0) - rejectInput("negative desired element quality"); + rejectInput("negative desired element quality", in->mesh->getPCU()); if (in->goodQuality > 1.0) - rejectInput("desired element quality greater than one"); + rejectInput("desired element quality greater than one", in->mesh->getPCU()); if (in->validQuality < 0.0) - rejectInput("negative minimum element quality"); + rejectInput("negative minimum element quality", in->mesh->getPCU()); if (in->maximumImbalance < 1.0) - rejectInput("maximum imbalance less than 1.0"); + rejectInput("maximum imbalance less than 1.0", in->mesh->getPCU()); if (in->maximumEdgeRatio < 1.0) - rejectInput("maximum tet edge ratio less than one"); + rejectInput("maximum tet edge ratio less than one", in->mesh->getPCU()); if (moreThanOneOptionIsTrue( in->shouldRunPreZoltan, in->shouldRunPreZoltanRib, in->shouldRunPreParma)) - rejectInput("only one of Zoltan, ZoltanRib, and Parma PreBalance options can be set to true!"); + rejectInput("only one of Zoltan, ZoltanRib, and Parma PreBalance options can be set to true!", in->mesh->getPCU()); if (moreThanOneOptionIsTrue( in->shouldRunPostZoltan, in->shouldRunPostZoltanRib, in->shouldRunPostParma)) - rejectInput("only one of Zoltan, ZoltanRib, and Parma PostBalance options can be set to true!"); + rejectInput("only one of Zoltan, ZoltanRib, and Parma PostBalance options can be set to true!", in->mesh->getPCU()); if (in->shouldRunMidZoltan && in->shouldRunMidParma) - rejectInput("only one of Zoltan and Parma MidBalance options can be set to true!"); + rejectInput("only one of Zoltan and Parma MidBalance options can be set to true!", in->mesh->getPCU()); #ifndef PUMI_HAS_ZOLTAN if (in->shouldRunPreZoltan || in->shouldRunPreZoltanRib || in->shouldRunMidZoltan) - rejectInput("core is not compiled with Zoltan. Use a different balancer or compile core with ENABLE_ZOLTAN=ON!"); + rejectInput("core is not compiled with Zoltan. Use a different balancer or compile core with ENABLE_ZOLTAN=ON!", in->mesh->getPCU()); #endif } @@ -155,13 +154,13 @@ static void updateMaxIterBasedOnSize(Mesh* m, Input* in) double maxMetricLength = getMaximumEdgeLength(m, in->sizeField); int iter = std::ceil(std::log2(maxMetricLength)); if (iter >= 10) { - print("ma::configure: Based on requested sizefield, MeshAdapt requires at least %d iterations,\n" + print(m->getPCU(), "ma::configure: Based on requested sizefield, MeshAdapt requires at least %d iterations,\n" " which is equal to or larger than the maximum of 10 allowed.\n" " Setting the number of iteration to 10!", iter); in->maximumIterations = 10; } else { - print("ma::configure: Based on requested sizefield, MeshAdapt requires at least %d iterations.\n" + print(m->getPCU(), "ma::configure: Based on requested sizefield, MeshAdapt requires at least %d iterations.\n" " Setting the number of iteration to %d!", iter, iter+1); in->maximumIterations = iter+1; } diff --git a/ma/maLayer.cc b/ma/maLayer.cc index bb214e8be..e53cd814c 100644 --- a/ma/maLayer.cc +++ b/ma/maLayer.cc @@ -1,4 +1,3 @@ -#include #include "maLayer.h" #include "maAdapt.h" #include "maRefine.h" @@ -38,7 +37,7 @@ static long markLayerElements(Adapt* a) } } } - n = PCU_Add_Long(n); + n = m->getPCU()->Add(n); a->hasLayer = (n != 0); if ( ! a->hasLayer) return 0; @@ -94,13 +93,13 @@ void unfreezeLayer(Adapt* a) void resetLayer(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); long n = markLayerElements(a); if (!n) return; freezeLayer(a); - double t1 = PCU_Time(); - print("marked %ld layer elements in %f seconds", n, t1 - t0); + double t1 = pcu::Time(); + print(a->mesh->getPCU(), "marked %ld layer elements in %f seconds", n, t1 - t0); } void findLayerBase(Adapt* a) @@ -149,7 +148,7 @@ void allowSplitInLayer(Adapt* a) if (getFlag(a,e,LAYER)) clearFlag(a,e,DONT_SPLIT); m->end(it); - print("allowing layer refinement"); + print(m->getPCU(), "allowing layer refinement"); } void collectForLayerRefine(Refine* r) @@ -166,7 +165,7 @@ void collectForLayerRefine(Refine* r) void checkLayerShape(Mesh* m, const char* key) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); Iterator* it = m->begin(m->getDimension()); Entity* e; long n = 0; @@ -187,9 +186,9 @@ void checkLayerShape(Mesh* m, const char* key) ++n; } m->end(it); - n = PCU_Add_Long(n); - double t1 = PCU_Time(); - print("%s: checked layer quality in %f seconds: %ld unsafe elements", key, t1 - t0, n); + n = m->getPCU()->Add(n); + double t1 = pcu::Time(); + print(m->getPCU(), "%s: checked layer quality in %f seconds: %ld unsafe elements", key, t1 - t0, n); } } diff --git a/ma/maLayerCoarsen.cc b/ma/maLayerCoarsen.cc index 12feced3d..0fe7670ca 100644 --- a/ma/maLayerCoarsen.cc +++ b/ma/maLayerCoarsen.cc @@ -1,4 +1,3 @@ -#include #include "maMesh.h" #include "maAdapt.h" #include "maLayer.h" @@ -41,7 +40,7 @@ static long markBaseEdgesToCollapse(Adapt* a) } } m->end(it); - return PCU_Add_Long(n); + return m->getPCU()->Add(n); } struct CurveLocalizer : public Crawler @@ -134,12 +133,12 @@ struct CurveLocalizer : public Crawler void send(Entity* v, int to) { int dest = getVertDest(v); - PCU_COMM_PACK(to, dest); + m->getPCU()->Pack(to, dest); } bool recv(Entity* v, int) { int dest; - PCU_COMM_UNPACK(dest); + m->getPCU()->Unpack(dest); return handle(v, dest); } Adapt* a; @@ -161,8 +160,8 @@ static apf::Migration* planLayerCollapseMigration(Adapt* a, int d, int round) (m->getModelType(m->toModel(e)) == d)) { Entity* v[2]; m->getDownward(e, 0, v); - cl.handle(v[0], PCU_Comm_Self()); - cl.handle(v[1], PCU_Comm_Self()); + cl.handle(v[0], m->getPCU()->Self()); + cl.handle(v[1], m->getPCU()->Self()); } m->end(it); crawlLayers(&cl); @@ -172,13 +171,13 @@ static apf::Migration* planLayerCollapseMigration(Adapt* a, int d, int round) static bool wouldEmptyParts(apf::Migration* plan) { apf::Mesh* m = plan->getMesh(); - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); size_t sendingAway = 0; for (int i = 0; i < plan->count(); ++i) if (plan->sending(plan->get(i)) != self) ++sendingAway; bool wouldEmptyThisPart = (sendingAway == m->count(m->getDimension())); - return PCU_Or(wouldEmptyThisPart); + return m->getPCU()->Or(wouldEmptyThisPart); } static void migrateForLayerCollapse(Adapt* a, int d, int round) @@ -258,8 +257,8 @@ static long collapseAllStacks(Adapt* a, int d) collapseLocalStacks(a, skipCount, successCount, failureCount, d); allSuccesses += successCount; ++round; - } while (PCU_Or(skipCount)); - return PCU_Add_Long(allSuccesses); + } while (a->mesh->getPCU()->Or(skipCount)); + return a->mesh->getPCU()->Add(allSuccesses); } bool coarsenLayer(Adapt* a) @@ -268,7 +267,7 @@ bool coarsenLayer(Adapt* a) return false; if ( ! a->input->shouldCoarsenLayer) return false; - double t0 = PCU_Time(); + double t0 = pcu::Time(); allowLayerToCollapse(a); findLayerBase(a); long count = markBaseEdgesToCollapse(a); @@ -283,8 +282,8 @@ bool coarsenLayer(Adapt* a) findIndependentSet(a); successCount += collapseAllStacks(a, d); } - double t1 = PCU_Time(); - print("coarsened %li layer edges in %f seconds",successCount,t1-t0); + double t1 = pcu::Time(); + print(m->getPCU(), "coarsened %li layer edges in %f seconds", successCount,t1-t0); resetLayer(a); return true; } diff --git a/ma/maLayerRefine.cc b/ma/maLayerRefine.cc index a8b13e6e2..2d2b3fe09 100644 --- a/ma/maLayerRefine.cc +++ b/ma/maLayerRefine.cc @@ -1,4 +1,3 @@ -#include #include "maCrawler.h" #include "maRefine.h" #include "maLayer.h" @@ -73,12 +72,12 @@ struct SplitTagger : public Crawler void send(Entity* e, int to) { bool has = getFlag(a, e, SPLIT); - PCU_COMM_PACK(to, has); + m->getPCU()->Pack(to, has); } bool recv(Entity* e, int) { bool has; - PCU_COMM_UNPACK(has); + m->getPCU()->Unpack(has); if (getFlag(a, e, CHECKED)) return false; handle(e, has); @@ -209,12 +208,12 @@ struct Disambiguator : public Crawler void send(Entity* t, int to) { int diag = getDiagonalFromFlag(a, t); - PCU_COMM_PACK(to, diag); + m->getPCU()->Pack(to, diag); } bool recv(Entity* t, int) { int diag; - PCU_COMM_UNPACK(diag); + m->getPCU()->Unpack(diag); if (getFlag(a, t, CHECKED)) return false; setFlag(a, t, CHECKED); diff --git a/ma/maLayerSnap.cc b/ma/maLayerSnap.cc index 1fc249800..23b1bb1dc 100644 --- a/ma/maLayerSnap.cc +++ b/ma/maLayerSnap.cc @@ -1,4 +1,3 @@ -#include #include "maCrawler.h" #include "maLayer.h" #include "maSnap.h" @@ -58,20 +57,20 @@ struct SnapTagger : public Crawler void send(Entity* v, int to) { bool has = m->hasTag(v, snapTag); - PCU_COMM_PACK(to, has); + m->getPCU()->Pack(to, has); if (has) { Vector s; m->getDoubleTag(v, snapTag, &s[0]); - PCU_COMM_PACK(to, s); + m->getPCU()->Pack(to, s); } } bool recv(Entity* v, int) { bool has; - PCU_COMM_UNPACK(has); + m->getPCU()->Unpack(has); Vector s; if (has) - PCU_COMM_UNPACK(s); + m->getPCU()->Unpack(s); if (getFlag(a, v, CHECKED)) return false; setFlag(a, v, CHECKED); @@ -134,7 +133,7 @@ struct BaseTopLinker : public Crawler void begin(Layer& first) { getDimensionBase(a, 0, first); - int peer = PCU_Comm_Self(); + int peer = m->getPCU()->Self(); for (size_t i = 0; i < first.size(); ++i) { if (!m->isOwned(first[i])) continue; @@ -160,12 +159,12 @@ struct BaseTopLinker : public Crawler { int link[2]; m->getIntTag(v, linkTag, link); - PCU_COMM_PACK(to, link); + m->getPCU()->Pack(to, link); } bool recv(Entity* v, int) { int link[2]; - PCU_COMM_UNPACK(link); + m->getPCU()->Unpack(link); if (hasLink(v)) return false; m->setIntTag(v, linkTag, link); @@ -234,7 +233,7 @@ struct LayerSnapper : public Crawler } } syncLayer(this, owned); - PCU_Add_Longs(&ncurves, 1); + m->getPCU()->Add(&ncurves, 1); } void end() { @@ -252,12 +251,12 @@ struct LayerSnapper : public Crawler void send(Entity* v, int to) { bool has = m->hasTag(v, snapTag); - PCU_COMM_PACK(to, has); + m->getPCU()->Pack(to, has); } bool recv(Entity* v, int) { bool has; - PCU_COMM_UNPACK(has); + m->getPCU()->Unpack(has); if (getFlag(a, v, CHECKED)) return false; handle(v, has); @@ -271,11 +270,11 @@ struct LayerSnapper : public Crawler static long snapAllCurves(Adapt* a, Tag* snapTag) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); LayerSnapper op(a, snapTag); crawlLayers(&op); - double t1 = PCU_Time(); - print("snapped %ld curves in %f seconds", op.ncurves, t1 - t0); + double t1 = pcu::Time(); + print(a->mesh->getPCU(), "snapped %ld curves in %f seconds", op.ncurves, t1 - t0); return op.ncurves; } @@ -352,12 +351,12 @@ struct UnsnapChecker : public Crawler void send(Entity* v, int to) { bool has = getFlag(a, v, LAYER_UNSNAP); - PCU_COMM_PACK(to, has); + m->getPCU()->Pack(to, has); } bool recv(Entity* v, int) { bool has; - PCU_COMM_UNPACK(has); + m->getPCU()->Unpack(has); bool wasChecked = getFlag(a, v, CHECKED); if (wasChecked) { if (has) @@ -387,7 +386,7 @@ static void crawlLayers_doubleSync(Crawler* c) { Crawler::Layer layer; c->begin(layer); - while (PCU_Or( ! layer.empty())) { + while (c->mesh->getPCU()->Or( ! layer.empty())) { crawlLayer(c, layer); syncLayer(c, layer); syncLayer(c, layer); @@ -397,15 +396,15 @@ static void crawlLayers_doubleSync(Crawler* c) static bool checkForUnsnap(Adapt* a, Tag* snapTag) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); UnsnapChecker op(a, snapTag); crawlLayers_doubleSync(&op); - bool notOk = PCU_Or(op.foundAnything); - double t1 = PCU_Time(); + bool notOk = a->mesh->getPCU()->Or(op.foundAnything); + double t1 = pcu::Time(); if (notOk) - print("checked snapped curves in %f seconds, found some to unsnap", t1 - t0); + print(a->mesh->getPCU(), "checked snapped curves in %f seconds, found some to unsnap", t1 - t0); else - print("checked snapped curves in %f seconds, all ok", t1 - t0); + print(a->mesh->getPCU(), "checked snapped curves in %f seconds, all ok", t1 - t0); return notOk; } @@ -417,7 +416,7 @@ static void feedbackUnsnap(Adapt* a, Tag* snapTag, BaseTopLinker& l) Mesh* m = l.m; long n = 0; Entity* v; - PCU_Comm_Begin(); + m->getPCU()->Begin(); Iterator* it = m->begin(0); while ((v = m->iterate(it))) if (getFlag(a, v, LAYER_TOP) && @@ -425,20 +424,20 @@ static void feedbackUnsnap(Adapt* a, Tag* snapTag, BaseTopLinker& l) m->isOwned(v)) { int peer, link; l.getLink(v, peer, link); - PCU_COMM_PACK(peer, link); + m->getPCU()->Pack(peer, link); ++n; } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { int link; - PCU_COMM_UNPACK(link); + m->getPCU()->Unpack(link); Entity* v = l.lookup(link); setFlag(a, v, LAYER_UNSNAP); PCU_ALWAYS_ASSERT(m->hasTag(v, snapTag)); } - n = PCU_Add_Long(n); - print("fed back unsnap flag from %ld tops", n); + n = m->getPCU()->Add(n); + print(m->getPCU(), "fed back unsnap flag from %ld tops", n); } /* for each layer curve whose base vertex @@ -486,7 +485,7 @@ struct Unsnapper : public Crawler ++ncurves; } } - PCU_Add_Longs(&ncurves, 1); + m->getPCU()->Add(&ncurves, 1); syncLayer(this, owned); } void end() @@ -506,12 +505,12 @@ struct Unsnapper : public Crawler void send(Entity* v, int to) { bool has = getFlag(a, v, LAYER_UNSNAP); - PCU_COMM_PACK(to, has); + m->getPCU()->Pack(to, has); } bool recv(Entity* v, int) { bool has; - PCU_COMM_UNPACK(has); + m->getPCU()->Unpack(has); if (getFlag(a, v, CHECKED)) return false; handle(v, has); @@ -525,11 +524,11 @@ struct Unsnapper : public Crawler static long unsnapMarkedCurves(Adapt* a, Tag* snapTag) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); Unsnapper op(a, snapTag); crawlLayers(&op); - double t1 = PCU_Time(); - print("unsnapped %ld curves in %f seconds", op.ncurves, t1 - t0); + double t1 = pcu::Time(); + print(a->mesh->getPCU(), "unsnapped %ld curves in %f seconds", op.ncurves, t1 - t0); return op.ncurves; } @@ -537,7 +536,7 @@ void snapLayer(Adapt* a, Tag* snapTag) { if ( ! a->hasLayer) return; - double t0 = PCU_Time(); + double t0 = pcu::Time(); findLayerBase(a); tagLayerForSnap(a, snapTag); flagLayerTop(a); @@ -550,8 +549,8 @@ void snapLayer(Adapt* a, Tag* snapTag) nunsnapped += unsnapMarkedCurves(a, snapTag); } delete l; - double t1 = PCU_Time(); - print("finished snapping %ld of %ld layer curves in %f seconds", + double t1 = pcu::Time(); + print(a->mesh->getPCU(), "finished snapping %ld of %ld layer curves in %f seconds", nsnapped - nunsnapped, nsnapped, t1 - t0); } diff --git a/ma/maMatch.cc b/ma/maMatch.cc index cd28b47bc..c17ecdda5 100644 --- a/ma/maMatch.cc +++ b/ma/maMatch.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "maMatch.h" #include "maMesh.h" #include "maAdapt.h" @@ -15,16 +14,16 @@ namespace ma { -static void packSplits(int to, EntityArray& splits) +static void packSplits(int to, EntityArray& splits, pcu::PCU *PCUObj) { - PCU_Comm_Pack(to, + PCUObj->Pack(to, static_cast(&(splits[0])), splits.getSize()*sizeof(Entity*)); } -static void unpackSplits(EntityArray& splits) +static void unpackSplits(EntityArray& splits, pcu::PCU *PCUObj) { - PCU_Comm_Unpack( + PCUObj->Unpack( static_cast(&(splits[0])), splits.getSize()*sizeof(Entity*)); } @@ -36,7 +35,7 @@ void matchNewElements(Refine* r) long face_count = 0; for (int d=1; d < m->getDimension(); ++d) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (size_t i=0; i < r->toSplit[d].getSize(); ++i) { Entity* e = r->toSplit[d][i]; @@ -49,31 +48,31 @@ void matchNewElements(Refine* r) { int to = matches[i].peer; Entity* match = matches[i].entity; - PCU_COMM_PACK(to,match); - packSplits(to,splits); + m->getPCU()->Pack(to,match); + packSplits(to,splits,m->getPCU()); } } - PCU_Comm_Send(); - while (PCU_Comm_Listen()) + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { - int from = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) + int from = m->getPCU()->Sender(); + while ( ! m->getPCU()->Unpacked()) { Entity* e; - PCU_COMM_UNPACK(e); + m->getPCU()->Unpack(e); int number; m->getIntTag(e,r->numberTag,&number); EntityArray& splits = r->newEntities[d][number]; EntityArray remoteSplits(splits.getSize()); - unpackSplits(remoteSplits); + unpackSplits(remoteSplits,m->getPCU()); for (size_t i=0; i < splits.getSize(); ++i) m->addMatch(splits[i],from,remoteSplits[i]); if (d==2) ++face_count; } } } - face_count = PCU_Add_Long(face_count); - print("updated matching for %li faces",face_count); + face_count = m->getPCU()->Add(face_count); + print(m->getPCU(), "updated matching for %li faces", face_count); } void preventMatchedCavityMods(Adapt* a) diff --git a/ma/maMatchedCollapse.cc b/ma/maMatchedCollapse.cc index c54bfe75e..c0f4091a9 100644 --- a/ma/maMatchedCollapse.cc +++ b/ma/maMatchedCollapse.cc @@ -10,7 +10,6 @@ #include "maMatchedCollapse.h" #include "maAdapt.h" #include -#include #include @@ -79,13 +78,13 @@ void Rebuilds::match(apf::Sharing* sh) apf::CopyArray orig_matches; sh->getCopies(orig, orig_matches); for (unsigned j = 0; j < orig_matches.getSize(); ++j) { - PCU_ALWAYS_ASSERT(orig_matches[j].peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(orig_matches[j].peer == mesh->getPCU()->Self()); Entity* gen_match_j = 0; for (unsigned k = 0; k < v.size(); ++k) if (v[k].original == orig_matches[j].entity) gen_match_j = v[k].e; PCU_ALWAYS_ASSERT(gen_match_j); - mesh->addMatch(gen, PCU_Comm_Self(), gen_match_j); + mesh->addMatch(gen, mesh->getPCU()->Self(), gen_match_j); } } } @@ -124,7 +123,7 @@ void MatchedCollapse::setEdges() apf::CopyArray copies; sharing->getCopies(e, copies); APF_ITERATE(apf::CopyArray, copies, it) { - PCU_ALWAYS_ASSERT(it->peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(it->peer == mesh->getPCU()->Self()); PCU_ALWAYS_ASSERT(it->entity != e); } collapses.setSize(copies.getSize() + 1); @@ -154,7 +153,7 @@ bool MatchedCollapse::checkTopo2() unsigned j; bool ok = false; for (j = 0; j < copies.getSize(); ++j) { - PCU_ALWAYS_ASSERT(copies[j].peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(copies[j].peer == mesh->getPCU()->Self()); if (copies[j].entity == collapses[i].vertToCollapse) { ok = true; break; diff --git a/ma/maMatchedSnapper.cc b/ma/maMatchedSnapper.cc index 87ce2c384..c5e87feb1 100644 --- a/ma/maMatchedSnapper.cc +++ b/ma/maMatchedSnapper.cc @@ -12,7 +12,6 @@ #include "maShapeHandler.h" #include #include -#include #include namespace ma { @@ -58,7 +57,7 @@ void MatchedSnapper::setVerts() apf::CopyArray copies; sharing->getCopies(v, copies); APF_ITERATE(apf::CopyArray, copies, it) { - PCU_ALWAYS_ASSERT(it->peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(it->peer == adapter->mesh->getPCU()->Self()); PCU_ALWAYS_ASSERT(it->entity != v); } diff --git a/ma/maMesh.cc b/ma/maMesh.cc index 7d9b5876f..3e0f370e8 100644 --- a/ma/maMesh.cc +++ b/ma/maMesh.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "maMesh.h" #include "maTables.h" #include @@ -280,8 +279,8 @@ void getBoundingBox(Mesh* m, Vector& lower, Vector& upper) lower.toArray(a); double b[3]; upper.toArray(b); - PCU_Min_Doubles(a, 3); - PCU_Max_Doubles(b, 3); + m->getPCU()->Min(a, 3); + m->getPCU()->Max(b, 3); lower.fromArray(a); upper.fromArray(b); } @@ -303,7 +302,7 @@ Vector getCentroid(Mesh* m) } m->end(it); pointSum.toArray(values); - PCU_Add_Doubles(&(values[0]),4); + m->getPCU()->Add(&(values[0]),4); return Vector(values)/values[3]; } @@ -370,7 +369,7 @@ double getAverageElementSize(Mesh* m) m->end(it); double& count = sums[1]; count = m->count(m->getDimension()); - PCU_Add_Doubles(sums,2); + m->getPCU()->Add(sums,2); return sizeSum / count; } @@ -385,7 +384,7 @@ double getMinimumElementSize(Mesh* m) if (size < minimum) minimum=size; } m->end(it); - return PCU_Min_Double(minimum); + return m->getPCU()->Min(minimum); } void getFaceEdgesAndDirections( diff --git a/ma/maRefine.cc b/ma/maRefine.cc index df7f5c0ed..915d8481a 100644 --- a/ma/maRefine.cc +++ b/ma/maRefine.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "maRefine.h" #include "maTemplates.h" #include "maAdapt.h" @@ -246,12 +245,12 @@ void splitElement(Refine* r, Entity* e) static void linkNewVerts(Refine* r) { - if (PCU_Comm_Peers()==1) - return; - struct { Entity* parent; Entity* vert; } message; Adapt* a = r->adapt; Mesh* m = a->mesh; - PCU_Comm_Begin(); + if (m->getPCU()->Peers()==1) + return; + struct { Entity* parent; Entity* vert; } message; + m->getPCU()->Begin(); for (int d=1; d < m->getDimension(); ++d) for (size_t i=0; i < r->newEntities[d].getSize(); ++i) { @@ -267,16 +266,16 @@ static void linkNewVerts(Refine* r) { int to = rp->first; message.parent = rp->second; - PCU_COMM_PACK(to,message); + m->getPCU()->Pack(to,message); } } - PCU_Comm_Send(); - while (PCU_Comm_Listen()) + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { - int from = PCU_Comm_Sender(); - while ( ! PCU_Comm_Unpacked()) + int from = m->getPCU()->Sender(); + while ( ! m->getPCU()->Unpacked()) { - PCU_COMM_UNPACK(message); + m->getPCU()->Unpack(message); Entity* v = findSplitVert(r,message.parent); m->addRemote(v,from,message.vert); } @@ -403,7 +402,8 @@ long markEdgesToSplit(Adapt* a) void processNewElements(Refine* r) { linkNewVerts(r); - if (PCU_Comm_Peers()>1) { + Mesh* m = r->adapt->mesh; + if (m->getPCU()->Peers()>1) { apf::stitchMesh(r->adapt->mesh); r->adapt->mesh->acceptChanges(); } @@ -419,7 +419,7 @@ void cleanupAfter(Refine* r) bool refine(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); --(a->refinesLeft); setupLayerForSplit(a); long count = markEdgesToSplit(a); @@ -438,8 +438,8 @@ bool refine(Adapt* a) processNewElements(r); destroySplitElements(r); forgetNewEntities(r); - double t1 = PCU_Time(); - print("refined %li edges in %f seconds",count,t1-t0); + double t1 = pcu::Time(); + print(a->mesh->getPCU(),"refined %li edges in %f seconds",count,t1-t0); resetLayer(a); if (a->hasLayer) checkLayerShape(a->mesh, "after refinement"); diff --git a/ma/maShape.cc b/ma/maShape.cc index c3479a07c..f34ebcc29 100644 --- a/ma/maShape.cc +++ b/ma/maShape.cc @@ -8,7 +8,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "maShape.h" #include "maSize.h" #include "maAdapt.h" @@ -166,7 +165,7 @@ double getMinQuality(Adapt* a) minqual = qual; } m->end(it); - return PCU_Min_Double(minqual); + return m->getPCU()->Min(minqual); } class ShortEdgeFixer : public Operator @@ -740,10 +739,10 @@ class QualityImprover2D : public Operator static double fixShortEdgeElements(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); ShortEdgeFixer fixer(a); applyOperator(a,&fixer); - double t1 = PCU_Time(); + double t1 = pcu::Time(); return t1 - t0; } @@ -779,12 +778,12 @@ static void improveQualities2D(Adapt* a) static double fixLargeAngles(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); if (a->mesh->getDimension()==3) fixLargeAngleTets(a); else fixLargeAngleTris(a); - double t1 = PCU_Time(); + double t1 = pcu::Time(); return t1 - t0; } @@ -798,12 +797,12 @@ static void alignLargeAngles(Adapt* a) double improveQualities(Adapt* a) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); if (a->mesh->getDimension() == 3) return 0; // TODO: implement this for 3D else improveQualities2D(a); - double t1 = PCU_Time(); + double t1 = pcu::Time(); return t1 - t0; } @@ -811,7 +810,7 @@ void fixElementShapes(Adapt* a) { if ( ! a->input->shouldFixShape) return; - double t0 = PCU_Time(); + double t0 = pcu::Time(); int count = markBadQuality(a); int originalCount = count; int prev_count; @@ -821,7 +820,7 @@ void fixElementShapes(Adapt* a) if ( ! count) break; prev_count = count; - print("--iter %d of shape correction loop: #bad elements %d", iter, count); + print(a->mesh->getPCU(), "--iter %d of shape correction loop: #bad elements %d", iter, count); time = fixLargeAngles(a); /* We need to snap the new verts as soon as they are * created (to avoid future problems). At the moment @@ -831,20 +830,20 @@ void fixElementShapes(Adapt* a) if (a->mesh->getDimension() == 3) snap(a); count = markBadQuality(a); - print("--fixLargeAngles in %f seconds: #bad elements %d",time,count); + print(a->mesh->getPCU(), "--fixLargeAngles in %f seconds: #bad elements %d", time,count); time = fixShortEdgeElements(a); count = markBadQuality(a); - print("--fixShortEdgeElements in %f seconds: #bad elements %d",time,count); + print(a->mesh->getPCU(), "--fixShortEdgeElements in %f seconds: #bad elements %d", time,count); if (count >= prev_count) unMarkBadQuality(a); // to make sure markEntities does not complain! // balance the mesh to avoid empty parts midBalance(a); - print("--percent change in number of bad elements %f", + print(a->mesh->getPCU(), "--percent change in number of bad elements %f", ((double) prev_count - (double) count) / (double) prev_count); iter++; } while(count < prev_count); - double t1 = PCU_Time(); - print("bad shapes down from %d to %d in %f seconds", + double t1 = pcu::Time(); + print(a->mesh->getPCU(), "bad shapes down from %d to %d in %f seconds", originalCount,count,t1-t0); } @@ -853,7 +852,7 @@ void alignElements(Adapt* a) int max_iter = 5; if ( ! a->input->shouldFixShape) return; - double t0 = PCU_Time(); + double t0 = pcu::Time(); int count = markBadQuality(a); int originalCount = count; int prev_count; @@ -869,8 +868,8 @@ void alignElements(Adapt* a) unMarkBadQuality(a); } while(count < prev_count && i < max_iter); - double t1 = PCU_Time(); - print("non-aligned elements down from %d to %d in %f seconds", + double t1 = pcu::Time(); + print(a->mesh->getPCU(), "non-aligned elements down from %d to %d in %f seconds", originalCount,count,t1-t0); } @@ -879,7 +878,7 @@ void printQuality(Adapt* a) if ( ! a->input->shouldPrintQuality) return; double minqual = getMinQuality(a); - print("worst element quality is %e", minqual); + print(a->mesh->getPCU(), "worst element quality is %e", minqual); } } diff --git a/ma/maSize.cc b/ma/maSize.cc index 1dea70eab..7f097d2eb 100644 --- a/ma/maSize.cc +++ b/ma/maSize.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "maSize.h" #include "maSolutionTransferHelper.h" #include "apfMatrix.h" @@ -667,7 +666,7 @@ double getAverageEdgeLength(Mesh* m) edge_count += 1.0; } m->end(it); - PCU_Add_Doubles(sums,2); + m->getPCU()->Add(sums,2); return length_sum / edge_count; } @@ -687,7 +686,7 @@ double getMaximumEdgeLength(Mesh* m, SizeField* sf) maxLength = length; } m->end(it); - PCU_Max_Doubles(&maxLength,1); + m->getPCU()->Max(&maxLength,1); return maxLength; } diff --git a/ma/maSnap.cc b/ma/maSnap.cc index 195dca005..40bb54d75 100644 --- a/ma/maSnap.cc +++ b/ma/maSnap.cc @@ -7,7 +7,6 @@ of the SCOREC Non-Commercial License this program is distributed under. *******************************************************************************/ -#include #include "maSnap.h" #include "maAdapt.h" #include "maOperator.h" @@ -780,8 +779,8 @@ bool snapAllVerts(Adapt* a, Tag* t, bool isSimple, long& successCount) { SnapAll op(a, t, isSimple); applyOperator(a, &op); - successCount += PCU_Add_Long(op.successCount); - return PCU_Or(op.didAnything); + successCount += a->mesh->getPCU()->Add(op.successCount); + return a->mesh->getPCU()->Or(op.didAnything); } class SnapMatched : public Operator @@ -831,8 +830,8 @@ bool snapMatchedVerts(Adapt* a, Tag* t, bool isSimple, long& successCount) { SnapMatched op(a, t, isSimple); applyOperator(a, &op); - successCount += PCU_Add_Long(op.successCount); - return PCU_Or(op.didAnything); + successCount += a->mesh->getPCU()->Add(op.successCount); + return a->mesh->getPCU()->Or(op.didAnything); } long tagVertsToSnap(Adapt* a, Tag*& t) @@ -857,7 +856,7 @@ long tagVertsToSnap(Adapt* a, Tag*& t) ++n; } m->end(it); - return PCU_Add_Long(n); + return m->getPCU()->Add(n); } static void markVertsToSnap(Adapt* a, Tag* t) @@ -902,7 +901,7 @@ void snap(Adapt* a) { if ( ! a->input->shouldSnap) return; - double t0 = PCU_Time(); + double t0 = pcu::Time(); Tag* tag; /* we are starting to support a few operations on matched meshes, including snapping+UR. this should prevent snapping @@ -913,8 +912,8 @@ void snap(Adapt* a) snapLayer(a, tag); apf::removeTagFromDimension(a->mesh, tag, 0); a->mesh->destroyTag(tag); - double t1 = PCU_Time(); - print("snapped in %f seconds: %ld targets, %ld non-layer snaps", + double t1 = pcu::Time(); + print(a->mesh->getPCU(), "snapped in %f seconds: %ld targets, %ld non-layer snaps", t1 - t0, targets, success); if (a->hasLayer) checkLayerShape(a->mesh, "after snapping"); diff --git a/ma/maTetrahedronize.cc b/ma/maTetrahedronize.cc index b497e3201..36b62f64f 100644 --- a/ma/maTetrahedronize.cc +++ b/ma/maTetrahedronize.cc @@ -1,4 +1,3 @@ -#include #include #include "maTetrahedronize.h" #include "maCrawler.h" @@ -153,12 +152,12 @@ struct QuadFlagger : public Crawler void send(Entity* e, int to) { int diagonal = getDiagonalFromFlag(adapter, e); - PCU_COMM_PACK(to, diagonal); + mesh->getPCU()->Pack(to, diagonal); } bool recv(Entity* e, int) { int diagonal; - PCU_COMM_UNPACK(diagonal); + mesh->getPCU()->Unpack(diagonal); if (getFlag(adapter, e, DIAGONAL_1 | DIAGONAL_2)) return false; setFlag(adapter, e, getFlagFromDiagonal(diagonal)); @@ -498,13 +497,13 @@ void tetrahedronize(Adapt* a) if ( ! a->input->shouldTurnLayerToTets) return; PCU_ALWAYS_ASSERT(a->hasLayer); - double t0 = PCU_Time(); + double t0 = pcu::Time(); prepareLayerToTets(a); Refine* r = a->refine; addAllLayerElements(r); tetrahedronizeCommon(r); - double t1 = PCU_Time(); - print("boundary layer converted to tets in %f seconds",t1-t0); + double t1 = pcu::Time(); + print(a->mesh->getPCU(), "boundary layer converted to tets in %f seconds", t1-t0); } /* start of the island pyramid cleanup system, @@ -620,7 +619,7 @@ static long markIslandPyramids(Adapt* a) } } m->end(it); - return PCU_Add_Long(n); + return m->getPCU()->Add(n); } static int countEntitiesWithFlag(Adapt* a, int flag, int dim) @@ -673,17 +672,17 @@ void cleanupLayer(Adapt* a) return; if (!a->input->shouldCleanupLayer) return; - double t0 = PCU_Time(); + double t0 = pcu::Time(); long n = prepareIslandCleanup(a); if (!n) { - print("no island pyramids found"); + print(a->mesh->getPCU(), "no island pyramids found"); return; } Refine* r = a->refine; addIslandPyramids(r); tetrahedronizeCommon(r); - double t1 = PCU_Time(); - print("tetrahedronized %ld island pyramids in %f seconds", n, t1-t0); + double t1 = pcu::Time(); + print(a->mesh->getPCU(), "tetrahedronized %ld island pyramids in %f seconds", n, t1-t0); } } diff --git a/mds/apfBox.cc b/mds/apfBox.cc index b6dec78c0..37fe0f2b6 100644 --- a/mds/apfBox.cc +++ b/mds/apfBox.cc @@ -67,7 +67,7 @@ int Grid::in(Indices is) BoxBuilder::BoxBuilder(int nx, int ny, int nz, double wx, double wy, double wz, - bool is): + bool is, pcu::PCU *PCUObj): grid(nx + 1, ny + 1, nz + 1), mgrid(nx ? 3 : 1, ny ? 3 : 1, nz ? 3 : 1) { @@ -78,7 +78,7 @@ BoxBuilder::BoxBuilder(int nx, int ny, int nz, is_simplex = is; formModelTable(); gmi_model* gm = buildModel(); - m = makeEmptyMdsMesh(gm, dim, false); + m = makeEmptyMdsMesh(gm, dim, false, PCUObj); v.resize(grid.total()); buildMeshAndModel(); } @@ -329,17 +329,17 @@ void BoxBuilder::buildMeshAndModel() Mesh2* makeMdsBox( int nex, int ney, int nez, - double wx, double wy, double wz, bool is) + double wx, double wy, double wz, bool is, pcu::PCU *PCUObj) { - BoxBuilder bb(nex, ney, nez, wx, wy, wz, is); + BoxBuilder bb(nex, ney, nez, wx, wy, wz, is, PCUObj); return bb.m; } gmi_model* makeMdsBoxModel( int nex, int ney, int nez, - double wx, double wy, double wz, bool is) + double wx, double wy, double wz, bool is, pcu::PCU *PCUObj) { - BoxBuilder bb(nex, ney, nez, wx, wy, wz, is); + BoxBuilder bb(nex, ney, nez, wx, wy, wz, is, PCUObj); return bb.buildModel(); } diff --git a/mds/apfBox.h b/mds/apfBox.h index 035a29b78..5fa443ef7 100644 --- a/mds/apfBox.h +++ b/mds/apfBox.h @@ -39,7 +39,7 @@ struct BoxBuilder std::vector v; BoxBuilder(int nx, int ny, int nz, double wx, double wy, double wz, - bool is); + bool is, pcu::PCU *PCUObj); void formModelTable(); void addModelUse(gmi_base* gb, agm_bdry ab, Indices di); gmi_model* buildModel(); @@ -71,11 +71,11 @@ struct BoxBuilder \param is true = simplical mesh, false = quad/hex \details set ny,nz=0 for a 1D mesh, set nz=0 for a 2D mesh */ Mesh2* makeMdsBox( - int nx, int ny, int nz, double wx, double wy, double wz, bool is); + int nx, int ny, int nz, double wx, double wy, double wz, bool is, pcu::PCU *PCUObj); /** \brief see makeMdsBox - only creates geometric model */ gmi_model* makeMdsBoxModel( - int nx, int ny, int nz, double wx, double wy, double wz, bool is); + int nx, int ny, int nz, double wx, double wy, double wz, bool is, pcu::PCU *PCUObj); } diff --git a/mds/apfMDS.cc b/mds/apfMDS.cc index 4c082f4c7..3b1c9bea9 100644 --- a/mds/apfMDS.cc +++ b/mds/apfMDS.cc @@ -167,9 +167,9 @@ class MeshMDS : public Mesh2 isMatched = false; ownsModel = false; } - MeshMDS(gmi_model* m, int d, bool isMatched_) + MeshMDS(gmi_model* m, int d, bool isMatched_, pcu::PCU *PCUObj) { - init(apf::getLagrange(1)); + init(apf::getLagrange(1), PCUObj); mds_id cap[MDS_TYPES] = {}; mesh = mds_apf_create(m, d, cap); isMatched = isMatched_; @@ -178,7 +178,7 @@ class MeshMDS : public Mesh2 MeshMDS(gmi_model* m, Mesh* from, apf::MeshEntity** nodes, apf::MeshEntity** elems, bool copy_data=true) { - init(apf::getLagrange(1)); + init(apf::getLagrange(1), from->getPCU()); mds_id cap[MDS_TYPES]; cap[MDS_VERTEX] = from->count(0); cap[MDS_EDGE] = from->count(1); @@ -195,11 +195,11 @@ class MeshMDS : public Mesh2 apf::convert(from, this, nodes, elems, copy_data); } - MeshMDS(gmi_model* m, const char* pathname) + MeshMDS(gmi_model* m, const char* pathname, pcu::PCU *PCUObj) { - init(apf::getLagrange(1)); - mesh = mds_read_smb(m, pathname, 0, this); - isMatched = PCU_Or(!mds_net_empty(&mesh->matches)); + init(apf::getLagrange(1), PCUObj); + mesh = mds_read_smb(getPCU()->GetCHandle(), m, pathname, 0, this); + isMatched = getPCU()->Or(!mds_net_empty(&mesh->matches)); ownsModel = true; } ~MeshMDS() @@ -579,14 +579,14 @@ class MeshMDS : public Mesh2 } int getId() { - return PCU_Comm_Self(); + return getPCU()->Self(); } void writeNative(const char* fileName) { - double t0 = PCU_Time(); - mesh = mds_write_smb(mesh, fileName, 0, this); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t0 = pcu::Time(); + mesh = mds_write_smb(getPCU()->GetCHandle(), mesh, fileName, 0, this); + double t1 = pcu::Time(); + if (!getPCU()->Self()) lion_oprint(1,"mesh %s written in %f seconds\n", fileName, t1 - t0); } void destroyNative() @@ -777,9 +777,9 @@ class MeshMDS : public Mesh2 bool ownsModel; }; -Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched) +Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched, pcu::PCU *PCUObj) { - Mesh2* m = new MeshMDS(model, dim, isMatched); + Mesh2* m = new MeshMDS(model, dim, isMatched, PCUObj); initResidence(m, dim); return m; } @@ -921,41 +921,42 @@ Mesh2* createMdsMesh(gmi_model* model, Mesh* from, bool reorder, bool copy_data) return new MeshMDS(model, from, &(node_arr[0]), &(elem_arr[0]), copy_data); } -Mesh2* loadSerialMdsMesh(gmi_model* model, const char* meshfile) +Mesh2* loadSerialMdsMesh(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj) { - Mesh2* m = new MeshMDS(model, meshfile); + Mesh2* m; + m = new MeshMDS(model, meshfile, PCUObj); return m; } -Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile) +Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj) { - double t0 = PCU_Time(); - Mesh2* m = new MeshMDS(model, meshfile); + double t0 = pcu::Time(); + Mesh2* m = new MeshMDS(model, meshfile, PCUObj); initResidence(m, m->getDimension()); stitchMesh(m); m->acceptChanges(); - if (!PCU_Comm_Self()) - lion_oprint(1,"mesh %s loaded in %f seconds\n", meshfile, PCU_Time() - t0); + if (!m->getPCU()->Self()) + lion_oprint(1,"mesh %s loaded in %f seconds\n", meshfile, pcu::Time() - t0); printStats(m); warnAboutEmptyParts(m); return m; } -Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile) +Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile, pcu::PCU *PCUObj) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); static gmi_model* model; model = gmi_load(modelfile); - if (!PCU_Comm_Self()) - lion_oprint(1,"model %s loaded in %f seconds\n", modelfile, PCU_Time() - t0); + if (!PCUObj->Self()) + lion_oprint(1,"model %s loaded in %f seconds\n", modelfile, pcu::Time() - t0); - return loadMdsMesh(model, meshfile); + return loadMdsMesh(model, meshfile, PCUObj); } void reorderMdsMesh(Mesh2* mesh, MeshTag* t) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); MeshMDS* m = static_cast(mesh); mds_tag* vert_nums; if (t) { @@ -964,61 +965,62 @@ void reorderMdsMesh(Mesh2* mesh, MeshTag* t) } else { vert_nums = mds_number_verts_bfs(m->mesh); } - m->mesh = mds_reorder(m->mesh, 0, vert_nums); - if (!PCU_Comm_Self()) - lion_oprint(1,"mesh reordered in %f seconds\n", PCU_Time()-t0); + m->mesh = mds_reorder(mesh->getPCU()->GetCHandle(), m->mesh, 0, vert_nums); + if (!mesh->getPCU()->Self()) + lion_oprint(1,"mesh reordered in %f seconds\n", pcu::Time()-t0); } -Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount) + +Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount, pcu::PCU *expandedPCU) { - double t0 = PCU_Time(); - int self = PCU_Comm_Self(); - int outputPartCount = PCU_Comm_Peers(); + double t0 = pcu::Time(); + int self = expandedPCU->Self(); + int outputPartCount = expandedPCU->Peers(); apf::Expand expand(inputPartCount, outputPartCount); apf::Contract contract(inputPartCount, outputPartCount); bool isOriginal = contract.isValid(self); int dim; bool isMatched; - PCU_Comm_Begin(); + expandedPCU->Begin(); if (isOriginal) { PCU_ALWAYS_ASSERT(m != 0); dim = m->getDimension(); isMatched = m->hasMatching(); for (int i = self + 1; i < outputPartCount && !contract.isValid(i); ++i) { - PCU_COMM_PACK(i, dim); - PCU_COMM_PACK(i, isMatched); - packDataClone(m, i); + expandedPCU->Pack(i, dim); + expandedPCU->Pack(i, isMatched); + packDataClone(m, i, expandedPCU); } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { - PCU_COMM_UNPACK(dim); - PCU_COMM_UNPACK(isMatched); - m = makeEmptyMdsMesh(g, dim, isMatched); + expandedPCU->Send(); + while (expandedPCU->Receive()) { + expandedPCU->Unpack(dim); + expandedPCU->Unpack(isMatched); + m = makeEmptyMdsMesh(g, dim, isMatched, expandedPCU); unpackDataClone(m); } PCU_ALWAYS_ASSERT(m != 0); apf::remapPartition(m, expand); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) lion_oprint(1,"mesh expanded from %d to %d parts in %f seconds\n", inputPartCount, outputPartCount, t1 - t0); return m; } Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, - int factor) + int factor, pcu::PCU *PCUObj) { - m = expandMdsMesh(m, g, PCU_Comm_Peers() / factor); - double t0 = PCU_Time(); - if (PCU_Comm_Self() % factor != 0) + m = expandMdsMesh(m, g, PCUObj->Peers() / factor, PCUObj); + double t0 = pcu::Time(); + if (PCUObj->Self() % factor != 0) plan = new apf::Migration(m, m->findTag("apf_migrate")); m->migrate(plan); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!PCUObj->Self()) lion_oprint(1,"mesh migrated from %d to %d in %f seconds\n", - PCU_Comm_Peers() / factor, - PCU_Comm_Peers(), + PCUObj->Peers() / factor, + PCUObj->Peers(), t1 - t0); return m; } @@ -1028,13 +1030,13 @@ bool alignMdsMatches(Mesh2* in) if (!in->hasMatching()) return false; MeshMDS* m = static_cast(in); - return mds_align_matches(m->mesh); + return mds_align_matches(in->getPCU()->GetCHandle(), m->mesh); } bool alignMdsRemotes(Mesh2* in) { MeshMDS* m = static_cast(in); - return mds_align_remotes(m->mesh); + return mds_align_remotes(in->getPCU()->GetCHandle(), m->mesh); } void deriveMdsModel(Mesh2* in) @@ -1300,11 +1302,11 @@ void hackMdsAdjacency(Mesh2* in, MeshEntity* up, int i, MeshEntity* down) mds_hack_adjacent(&m->mesh->mds, fromEnt(up), i, fromEnt(down)); } -Mesh2* loadMdsPart(gmi_model* model, const char* meshfile) +Mesh2* loadMdsPart(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj) { MeshMDS* m = new MeshMDS(); - m->init(apf::getLagrange(1)); - m->mesh = mds_read_smb(model, meshfile, 1, m); + m->init(apf::getLagrange(1), PCUObj); + m->mesh = mds_read_smb(m->getPCU()->GetCHandle(), model, meshfile, 1, m); m->isMatched = false; m->ownsModel = true; initResidence(m, m->getDimension()); @@ -1314,7 +1316,7 @@ Mesh2* loadMdsPart(gmi_model* model, const char* meshfile) void writeMdsPart(Mesh2* in, const char* meshfile) { MeshMDS* m = static_cast(in); - m->mesh = mds_write_smb(m->mesh, meshfile, 1, m); + m->mesh = mds_write_smb(m->getPCU()->GetCHandle(), m->mesh, meshfile, 1, m); } diff --git a/mds/apfMDS.h b/mds/apfMDS.h index afb130fa2..47d4cfbaf 100644 --- a/mds/apfMDS.h +++ b/mds/apfMDS.h @@ -35,8 +35,14 @@ // AJP: including apf.h for single define: CGNSBCMap // AJP: alternative is to allow common cgns base header // but trying to avoid that since it's not core functionality -#include +#include +#include + // +namespace pcu{ + class PCU; + PCU* PCU_GetGlobal(); +} struct gmi_model; namespace apf { @@ -56,7 +62,7 @@ typedef std::map GlobalToVert; \param dim the eventual mesh dimension. MDS needs to allocate arrays based on this before users add entities. \param isMatched whether or not there will be matched entities */ -Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched); +Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched, pcu::PCU *PCUObj); /** \brief load an MDS mesh and model from file \param modelfile will be passed to gmi_load to get the model @@ -65,7 +71,7 @@ Mesh2* makeEmptyMdsMesh(gmi_model* model, int dim, bool isMatched); may also be called to enable loading of GeomSim, Parasolid, and ACIS models */ -Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile); +Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile, pcu::PCU *PCUObj); /** \brief load an MDS mesh from files \param model the geometric model interface @@ -80,10 +86,10 @@ Mesh2* loadMdsMesh(const char* modelfile, const char* meshfile); using PCU file IO functions. Calling apf::Mesh::writeNative on the resulting object will do the same in reverse. */ -Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile); +Mesh2* loadMdsMesh(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj); // make a serial mesh on all processes - no pmodel & remote link setup -Mesh2* loadSerialMdsMesh(gmi_model* model, const char* meshfile); +Mesh2* loadSerialMdsMesh(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj); /** \brief create an MDS mesh from an existing mesh \param from the mesh to copy @@ -108,9 +114,9 @@ Mesh2* createMdsMesh(gmi_model* model, Mesh* from, bool reorder=false, bool copy there are no gaps in the MDS arrays after this */ void reorderMdsMesh(Mesh2* mesh, MeshTag* t = 0); -Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, int factor); -Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount); +Mesh2* repeatMdsMesh(Mesh2* m, gmi_model* g, Migration* plan, int factor, pcu::PCU *PCUObj); +Mesh2* expandMdsMesh(Mesh2* m, gmi_model* g, int inputPartCount, pcu::PCU *expandedPCU); /** \brief align the downward adjacencies of matched entities */ bool alignMdsMatches(Mesh2* in); /** \brief align the downward adjacencies of remote copies */ @@ -198,21 +204,22 @@ int getMdsIndex(Mesh2* in, MeshEntity* e); so call apf::reorderMdsMesh after any mesh modification. */ MeshEntity* getMdsEntity(Mesh2* in, int dimension, int index); -Mesh2* loadMdsFromCGNS(gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap); +Mesh2* loadMdsFromCGNS(PCU_t h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap); // names of mesh data to read from file: (VERTEX, VelocityX; CellCentre, Pressure) -Mesh2* loadMdsFromCGNS(gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap, const std::vector>& meshData); +Mesh2* loadMdsFromCGNS(PCU_t h, gmi_model* g, const char* filename, CGNSBCMap& cgnsBCMap, const std::vector>& meshData); + int gmshMajorVersion(const char* filename); -Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename); +Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename, pcu::PCU *PCUObj); -Mesh2* loadMdsDmgFromGmsh(const char* fnameDmg, const char* filename); +Mesh2* loadMdsDmgFromGmsh(const char* fnameDmg, const char* filename, pcu::PCU *PCUObj); -Mesh2* loadMdsFromUgrid(gmi_model* g, const char* filename); +Mesh2* loadMdsFromUgrid(gmi_model* g, const char* filename, pcu::PCU *PCUObj); void printUgridPtnStats(gmi_model* g, const char* ugridfile, const char* ptnfile, - const double elmWeights[]); + const double elmWeights[], pcu::PCU *PCUObj); /** \brief load an MDS mesh from ANSYS .node and .elem files \details this call takes two filenames, one @@ -225,13 +232,13 @@ void printUgridPtnStats(gmi_model* g, const char* ugridfile, const char* ptnfile currently, ANSYS element types SOLID72 and SOLID92 are supported, which become linear and quadratic tetrahedra, respectively. */ -Mesh2* loadMdsFromANSYS(const char* nodefile, const char* elemfile); +Mesh2* loadMdsFromANSYS(const char* nodefile, const char* elemfile, pcu::PCU *PCUObj); void disownMdsModel(Mesh2* in); void setMdsMatching(Mesh2* in, bool has); -Mesh2* loadMdsPart(gmi_model* model, const char* meshfile); +Mesh2* loadMdsPart(gmi_model* model, const char* meshfile, pcu::PCU *PCUObj); void writeMdsPart(Mesh2* m, const char* meshfile); } diff --git a/mds/apfPM.cc b/mds/apfPM.cc index f15704af6..90039b153 100644 --- a/mds/apfPM.cc +++ b/mds/apfPM.cc @@ -8,7 +8,7 @@ *******************************************************************************/ -#include + #include "apfPM.h" #include #include @@ -95,14 +95,14 @@ static void getCountMap(apf::Mesh* m, PM& ps, CountMap& mp) size_t n; getAdjacentParts(m, ps, peers); n = m->count(m->getDimension()); - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(apf::Parts, peers, it) - PCU_COMM_PACK(*it, n); - PCU_Comm_Send(); + m->getPCU()->Pack(*it, n); + m->getPCU()->Send(); mp[m->getId()] = n; - while (PCU_Comm_Listen()) { - PCU_COMM_UNPACK(n); - mp[PCU_Comm_Sender()] = n; + while (m->getPCU()->Listen()) { + m->getPCU()->Unpack(n); + mp[m->getPCU()->Sender()] = n; } } diff --git a/mds/mds.c b/mds/mds.c index ed2e0cc51..a896da704 100644 --- a/mds/mds.c +++ b/mds/mds.c @@ -13,7 +13,6 @@ #include #include #include -#include #include #include diff --git a/mds/mdsANSYS.cc b/mds/mdsANSYS.cc index e27961bfc..3bd0a8eb6 100644 --- a/mds/mdsANSYS.cc +++ b/mds/mdsANSYS.cc @@ -90,7 +90,7 @@ static void parseNodes(const char* nodefile, Nodes& nodes) typedef std::map Vertices; -static Mesh2* parseElems(const char* elemfile, Nodes& nodes) +static Mesh2* parseElems(const char* elemfile, Nodes& nodes, pcu::PCU *PCUObj) { Mesh2* m = 0; std::ifstream f(elemfile); @@ -107,7 +107,8 @@ static Mesh2* parseElems(const char* elemfile, Nodes& nodes) apf::Numbering* enumbers = 0; while (parseElem(f, en, type, id, shape)) { if (!m) { - m = makeEmptyMdsMesh(gmi_load(".null"), Mesh::typeDimension[type], false); + m = makeEmptyMdsMesh(gmi_load(".null"), Mesh::typeDimension[type], false, PCUObj); + if (shape != m->getShape()) changeMeshShape(m, shape, false); enumbers = createNumbering(m, "ansys_element", getConstant(m->getDimension()), 1); @@ -151,11 +152,11 @@ static Mesh2* parseElems(const char* elemfile, Nodes& nodes) return m; } -Mesh2* loadMdsFromANSYS(const char* nodefile, const char* elemfile) +Mesh2* loadMdsFromANSYS(const char* nodefile, const char* elemfile, pcu::PCU *PCUObj) { Nodes nodes; parseNodes(nodefile, nodes); - Mesh2* m = parseElems(elemfile, nodes); + Mesh2* m = parseElems(elemfile, nodes, PCUObj); m->acceptChanges(); deriveMdsModel(m); return m; diff --git a/mds/mdsCGNS.cc b/mds/mdsCGNS.cc index acdef4aae..fd811f914 100644 --- a/mds/mdsCGNS.cc +++ b/mds/mdsCGNS.cc @@ -10,7 +10,7 @@ #include "apfShape.h" #include "gmi.h" /* this is for gmi_getline... */ #include -#include +#include #include #include #include "apfFieldData.h" @@ -177,19 +177,19 @@ struct MeshDataGroup if (components.size() == 1) { std::cout << "Scalar Group has " << components.size() << " related componenets: " << std::endl; - for (const auto m : components) + for (const auto& m : components) std::cout << "Field " << m.second.name << " @ " << m.second.si << " " << m.second.fi << std::endl; } else if (components.size() == 3) { std::cout << "Vector Group has " << components.size() << " related componenets: " << std::endl; - for (const auto m : components) + for (const auto& m : components) std::cout << "Field " << m.second.name << " @ " << m.second.si << " " << m.second.fi << std::endl; } else if (components.size() == 9) { std::cout << "Matrix Group has " << components.size() << " related componenets: " << std::endl; - for (const auto m : components) + for (const auto& m : components) std::cout << "Field " << m.second.name << " @ " << m.second.si << " " << m.second.fi << std::endl; } else @@ -200,14 +200,14 @@ struct MeshDataGroup }; template -void DebugParallelPrinter(std::ostream &out, Arg &&arg, Args &&... args) +void DebugParallelPrinter(PCU_t h, std::ostream &out, Arg &&arg, Args &&... args) { // if constexpr (debugOutput) // probably will not get away with c++17 if (debugOutput) { - for (int i = 0; i < PCU_Comm_Peers(); i++) + for (int i = 0; i < PCU_Comm_Peers(h); i++) { - if (i == PCU_Comm_Self()) + if (i == PCU_Comm_Self(h)) { out << "Rank [" << i << "] " << std::forward(arg); //((out << ", " << std::forward(args)), ...); // probably will not get away with c++17 @@ -223,15 +223,15 @@ void DebugParallelPrinter(std::ostream &out, Arg &&arg, Args &&... args) } template -void Kill(const int fid, Args &&... args) +void Kill(PCU_t h, const int fid, Args &&... args) { - DebugParallelPrinter(std::cout, "***** CGNS ERROR", args...); + DebugParallelPrinter(h, std::cout, "***** CGNS ERROR", args...); - if (PCU_Comm_Initialized()) + if (PCU_Comm_Initialized(h)) { cgp_close(fid); // Finalize the MPI environment. - PCU_Comm_Free(); + PCU_Comm_Free(&h); MPI_Finalize(); cgp_error_exit(); exit(EXIT_FAILURE); @@ -244,15 +244,15 @@ void Kill(const int fid, Args &&... args) } } -void Kill(const int fid) +void Kill(PCU_t h, const int fid) { - DebugParallelPrinter(std::cout, "***** CGNS ERROR"); + DebugParallelPrinter(h, std::cout, "***** CGNS ERROR"); - if (PCU_Comm_Initialized()) + if (PCU_Comm_Initialized(h)) { cgp_close(fid); // Finalize the MPI environment. - PCU_Comm_Free(); + PCU_Comm_Free(&h); MPI_Finalize(); cgp_error_exit(); exit(EXIT_FAILURE); @@ -265,13 +265,13 @@ void Kill(const int fid) } } -auto ReadCGNSCoords(int cgid, int base, int zone, int ncoords, int nverts, const std::vector &, const apf::GlobalToVert &globalToVert) +auto ReadCGNSCoords(PCU_t h, int cgid, int base, int zone, int ncoords, int nverts, const std::vector &, const apf::GlobalToVert &globalToVert) { // Read min required as defined by consecutive range // make one based as ReadElements makes zero based const cgsize_t lowest = globalToVert.begin()->first + 1; const cgsize_t highest = globalToVert.rbegin()->first + 1; - DebugParallelPrinter(std::cout, "From globalToVert ", lowest, highest, nverts); + DebugParallelPrinter(h, std::cout, "From globalToVert ", lowest, highest, nverts); cgsize_t range_min[3]; range_min[0] = lowest; @@ -301,7 +301,7 @@ auto ReadCGNSCoords(int cgid, int base, int zone, int ncoords, int nverts, const if (cg_coord_info(cgid, base, zone, d + 1, &datatype, &coord_names[d][0])) { std::cout << __LINE__ << " CGNS is dead " << std::endl; - Kill(cgid); + Kill(h, cgid); } const auto coord_name = std::string(coord_names[d].c_str()); //boost::algorithm::trim(coord_name); // can't be bothered including boost @@ -311,7 +311,7 @@ auto ReadCGNSCoords(int cgid, int base, int zone, int ncoords, int nverts, const ordinate.data())) { std::cout << __LINE__ << " CGNS is dead " << std::endl; - Kill(cgid); + Kill(h, cgid); } } // to be clear, indices passed back are global, zero based @@ -332,14 +332,14 @@ auto ReadCGNSCoords(int cgid, int base, int zone, int ncoords, int nverts, const return points; } -void SimpleElementPartition(std::vector &numberToReadPerProc, std::vector &startingIndex, int el_start /* one based */, int el_end, int numElements) +void SimpleElementPartition(PCU_t h, std::vector &numberToReadPerProc, std::vector &startingIndex, int el_start /* one based */, int el_end, int numElements) { - numberToReadPerProc.resize(PCU_Comm_Peers(), 0); + numberToReadPerProc.resize(PCU_Comm_Peers(h), 0); const cgsize_t blockSize = static_cast( std::floor(static_cast(numElements) / - static_cast(PCU_Comm_Peers()))); + static_cast(PCU_Comm_Peers(h)))); - DebugParallelPrinter(std::cout, "BlockSize", blockSize, "numElements", numElements, "comms Size", PCU_Comm_Peers()); + DebugParallelPrinter(h, std::cout, "BlockSize", blockSize, "numElements", numElements, "comms Size", PCU_Comm_Peers(h)); cgsize_t counter = 0; if (blockSize == 0 && numElements > 0) @@ -349,7 +349,7 @@ void SimpleElementPartition(std::vector &numberToReadPerProc, std::vec bool keepGoing = true; while (keepGoing) { - for (int p = 0; p < PCU_Comm_Peers() && keepGoing; p++) + for (int p = 0; p < PCU_Comm_Peers(h) && keepGoing; p++) { if (counter + blockSize <= numElements) { @@ -366,11 +366,11 @@ void SimpleElementPartition(std::vector &numberToReadPerProc, std::vec } } } - DebugParallelPrinter(std::cout, "Sanity check: Counter", counter, "numElements", numElements); + DebugParallelPrinter(h, std::cout, "Sanity check: Counter", counter, "numElements", numElements); - DebugParallelPrinter(std::cout, "numberToReadPerProc for rank", PCU_Comm_Self(), "is:", numberToReadPerProc[PCU_Comm_Self()]); + DebugParallelPrinter(h, std::cout, "numberToReadPerProc for rank", PCU_Comm_Self(h), "is:", numberToReadPerProc[PCU_Comm_Self(h)]); - startingIndex.resize(PCU_Comm_Peers(), 0); + startingIndex.resize(PCU_Comm_Peers(h), 0); startingIndex[0] = el_start; for (std::size_t i = 1; i < numberToReadPerProc.size(); i++) { @@ -378,36 +378,36 @@ void SimpleElementPartition(std::vector &numberToReadPerProc, std::vec startingIndex[i] = startingIndex[i - 1] + numberToReadPerProc[i - 1]; } - DebugParallelPrinter(std::cout, "Element start, end, numElements ", el_start, + DebugParallelPrinter(h, std::cout, "Element start, end, numElements ", el_start, el_end, numElements); - DebugParallelPrinter(std::cout, "startingIndex for rank", PCU_Comm_Self(), "is:", startingIndex[PCU_Comm_Self()]); + DebugParallelPrinter(h, std::cout, "startingIndex for rank", PCU_Comm_Self(h), "is:", startingIndex[PCU_Comm_Self(h)]); - DebugParallelPrinter(std::cout, "Returning from SimpleElementPartition \n"); + DebugParallelPrinter(h, std::cout, "Returning from SimpleElementPartition \n"); } using Pair = std::pair; using LocalElementRanges = std::vector; // one based -auto ReadElements(int cgid, int base, int zone, int section, int el_start /* one based */, int el_end, int numElements, int verticesPerElement, LocalElementRanges &localElementRanges) +auto ReadElements(PCU_t h, int cgid, int base, int zone, int section, int el_start /* one based */, int el_end, int numElements, int verticesPerElement, LocalElementRanges &localElementRanges) { std::vector numberToReadPerProc; std::vector startingIndex; - SimpleElementPartition(numberToReadPerProc, startingIndex, el_start, el_end, numElements); + SimpleElementPartition(h, numberToReadPerProc, startingIndex, el_start, el_end, numElements); std::vector vertexIDs; - if (numberToReadPerProc[PCU_Comm_Self()] > 0) - vertexIDs.resize(numberToReadPerProc[PCU_Comm_Self()] * verticesPerElement, + if (numberToReadPerProc[PCU_Comm_Self(h)] > 0) + vertexIDs.resize(numberToReadPerProc[PCU_Comm_Self(h)] * verticesPerElement, -1234567); - const auto start = startingIndex[PCU_Comm_Self()]; - const auto end = startingIndex[PCU_Comm_Self()] + numberToReadPerProc[PCU_Comm_Self()] - 1; - DebugParallelPrinter(std::cout, "Range", start, "to", end, numberToReadPerProc[PCU_Comm_Self()]); + const auto start = startingIndex[PCU_Comm_Self(h)]; + const auto end = startingIndex[PCU_Comm_Self(h)] + numberToReadPerProc[PCU_Comm_Self(h)] - 1; + DebugParallelPrinter(h, std::cout, "Range", start, "to", end, numberToReadPerProc[PCU_Comm_Self(h)]); // cgp_elements_read_data(cgid, base, zone, section, start, end, vertexIDs.data()); - if (numberToReadPerProc[PCU_Comm_Self()] > 0) + if (numberToReadPerProc[PCU_Comm_Self(h)] > 0) { localElementRanges.push_back(std::make_pair(start, end)); } @@ -419,7 +419,7 @@ auto ReadElements(int cgid, int base, int zone, int section, int el_start /* one i = i - 1; } - return std::make_tuple(vertexIDs, numberToReadPerProc[PCU_Comm_Self()]); + return std::make_tuple(vertexIDs, numberToReadPerProc[PCU_Comm_Self(h)]); } struct CGNSBCMeta @@ -497,7 +497,7 @@ struct BCInfo } else { - Kill(cgid, "GlobalToVert lookup problem", v); + Kill(m->getPCU()->GetCHandle(), cgid, "GlobalToVert lookup problem", v); } } @@ -805,7 +805,7 @@ struct BCInfo Add("CellCenter", bcName, bcTag); } else - Kill(cgid, "Unknown BC Type", cgnsLocation); + Kill(m->getPCU()->GetCHandle(), cgid, "Unknown BC Type", cgnsLocation); } else if (m->getDimension() == 2) // working with a 2D mesh { @@ -856,7 +856,7 @@ struct BCInfo } }; // namespace -void ReadBCInfo(const int cgid, const int base, const int zone, const int nBocos, const int physDim, const int cellDim, const int nsections, std::vector &bcInfos, const apf::GlobalToVert &globalToVert) +void ReadBCInfo(PCU_t h, const int cgid, const int base, const int zone, const int nBocos, const int physDim, const int cellDim, const int nsections, std::vector &bcInfos, const apf::GlobalToVert &globalToVert) { // Read the BCS. std::vector bcMetas(nBocos); @@ -874,7 +874,7 @@ void ReadBCInfo(const int cgid, const int base, const int zone, const int nBocos if (cg_boco_info(cgid, base, zone, boco, &bcMeta.bocoName[0], &bcMeta.bocoType, &bcMeta.ptsetType, &bcMeta.npnts, bcMeta.normalindices.data(), &bcMeta.normalListSize, &bcMeta.normalDataType, &bcMeta.ndataset)) - Kill(cgid, "Failed cg_boco_info"); + Kill(h, cgid, "Failed cg_boco_info"); if (bcMeta.ptsetType == CGNS_ENUMV(PointList) || (bcMeta.ptsetType == CGNS_ENUMV(PointRange))) { @@ -882,7 +882,7 @@ void ReadBCInfo(const int cgid, const int base, const int zone, const int nBocos //boost::algorithm::trim(bcMeta.bocoName); // can't be bothered including boost if (cg_boco_gridlocation_read(cgid, base, zone, boco, &bcMeta.location)) - Kill(cgid, "Failed cg_boco_gridlocation_read"); + Kill(h, cgid, "Failed cg_boco_gridlocation_read"); bcMeta.locationName = cg_GridLocationName(bcMeta.location); @@ -891,7 +891,7 @@ void ReadBCInfo(const int cgid, const int base, const int zone, const int nBocos } else { - Kill(cgid, + Kill(h, cgid, "TODO: Can only work with " "PointList and PointRange BC " "types at the moment"); @@ -907,25 +907,25 @@ void ReadBCInfo(const int cgid, const int base, const int zone, const int nBocos if (bcMeta.locationName == "Vertex") // && cellDim == 1) { if (cg_boco_read(cgid, base, zone, boco, bcMeta.bcElementIds.data(), NULL)) - Kill(cgid, "Failed cg_boco_read"); + Kill(h, cgid, "Failed cg_boco_read"); } else if (bcMeta.locationName == "EdgeCenter") // && cellDim == 2) { if (cg_boco_read(cgid, base, zone, boco, bcMeta.bcElementIds.data(), NULL)) - Kill(cgid, "Failed cg_boco_read"); + Kill(h, cgid, "Failed cg_boco_read"); } else if (bcMeta.locationName == "FaceCenter") // && cellDim == 3) { if (cg_boco_read(cgid, base, zone, boco, bcMeta.bcElementIds.data(), NULL)) - Kill(cgid, "Failed cg_boco_read"); + Kill(h, cgid, "Failed cg_boco_read"); } else if (bcMeta.locationName == "CellCenter") // && cellDim == 3) { if (cg_boco_read(cgid, base, zone, boco, bcMeta.bcElementIds.data(), NULL)) - Kill(cgid, "Failed cg_boco_read"); + Kill(h, cgid, "Failed cg_boco_read"); } else - Kill(cgid, "Failed Location test for BC Type", bcMeta.locationName, + Kill(h, cgid, "Failed Location test for BC Type", bcMeta.locationName, cellDim); if (pointRange) @@ -1049,15 +1049,15 @@ void ReadBCInfo(const int cgid, const int base, const int zone, const int nBocos } } -apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &readMeshData) +apf::Mesh2 *DoIt(PCU_t h, gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &readMeshData) { static_assert(std::is_same::value, "cgsize_t not compiled as int"); int cgid = -1; - auto comm = PCU_Get_Comm(); + auto comm = PCU_Get_Comm(h); cgp_mpi_comm(comm); - cgp_pio_mode(CGNS_ENUMV(CGP_INDEPENDENT)); - cgp_open(fname.c_str(), CGNS_ENUMV(CG_MODE_READ), &cgid); + cgp_pio_mode(CGP_INDEPENDENT); + cgp_open(fname.c_str(), CG_MODE_READ, &cgid); int nbases = -1; cg_nbases(cgid, &nbases); @@ -1076,7 +1076,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM const int readDim = cellDim; // Salome cgns is a bit on the odd side: cellDim, physDim, ncoords are not always consistent - apf::Mesh2 *mesh = apf::makeEmptyMdsMesh(g, cellDim, false); + apf::Mesh2 *mesh = apf::makeEmptyMdsMesh(g, cellDim, false, static_cast(h.ptr)); apf::GlobalToVert globalToVert; LocalElementRanges localElementRanges; @@ -1111,7 +1111,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM if (ngrids > 1) { std::cout << __LINE__ << " CGNS is dead " << std::endl; - Kill(cgid); + Kill(h, cgid); } int ncoords = -1; cg_ncoords(cgid, base, zone, &ncoords); @@ -1128,7 +1128,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM else { std::cout << __LINE__ << " CGNS is dead " << std::endl; - Kill(cgid); + Kill(h, cgid); } int nBocos = -1; @@ -1157,14 +1157,14 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM cg_npe(elementType, &verticesPerElement); const auto readElementsAndVerts = [&](const apf::Mesh2::Type &type) { - const auto &ret = ReadElements(cgid, base, zone, section, el_start, el_end, numElements, verticesPerElement, localElementRanges); + const auto &ret = ReadElements(h, cgid, base, zone, section, el_start, el_end, numElements, verticesPerElement, localElementRanges); if (std::get<1>(ret) > 0) { const std::vector vertexIDs = std::get<0>(ret); std::vector vertexIDs_l(vertexIDs.begin(), vertexIDs.end()); localElements.emplace_back(apf::assemble(mesh, vertexIDs_l.data(), std::get<1>(ret), type, globalToVert)); // corresponding finalize below const auto nverts = sizes[0]; - const auto ordinates = ReadCGNSCoords(cgid, base, zone, ncoords, nverts, vertexIDs, globalToVert); + const auto ordinates = ReadCGNSCoords(h, cgid, base, zone, ncoords, nverts, vertexIDs, globalToVert); for (const auto &p : globalToVert) { @@ -1220,7 +1220,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM { std::cout << __LINE__ << " CGNS is dead " << " " << SupportedCGNSElementTypeToString(elementType) << std::endl; - Kill(cgid); + Kill(h, cgid); } } @@ -1229,13 +1229,13 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM std::cout << std::endl; std::cout << "Attempting to read BCS info " << " " << nBocos << std::endl; - ReadBCInfo(cgid, base, zone, nBocos, physDim, cellDim, nsections, bcInfos, globalToVert); + ReadBCInfo(h, cgid, base, zone, nBocos, physDim, cellDim, nsections, bcInfos, globalToVert); std::cout << std::endl; } int nsols = -1; if (cg_nsols(cgid, base, zone, &nsols)) - Kill(cgid, "1, ", nsols); + Kill(h, cgid, "1, ", nsols); std::vector meshData; if (nsols > 0) @@ -1247,11 +1247,11 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM char sname[33]; if (cg_sol_info(cgid, base, zone, ns, sname, &location)) - Kill(cgid, "2"); + Kill(h, cgid, "2"); int nflds = -1; if (cg_nfields(cgid, base, zone, ns, &nflds)) - Kill(cgid, "3"); + Kill(h, cgid, "3"); if (nflds > 0) { @@ -1262,7 +1262,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM char name[33]; if (cg_field_info(cgid, base, zone, ns, f, &datatype, name)) - Kill(cgid, "4"); + Kill(h, cgid, "4"); //std::cout << sname << " " << name << " " << f << " " << ns << " " << cg_DataTypeName(datatype) << " " << cg_GridLocationName(location) << std::endl; meshData.push_back(MeshData(ns, f, datatype, location, name)); @@ -1286,7 +1286,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM }; // process meshData and find vectors (and matrices) - const auto findMatch = [&meshData, &index, &cgid](MeshData &other, std::vector &meshDataGroups) { + const auto findMatch = [&meshData, &index, &cgid, &h](MeshData &other, std::vector &meshDataGroups) { MeshDataGroup group; for (auto &md : meshData) { @@ -1318,7 +1318,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM if (otherIndex != -1) group.insert(otherIndex, other); else - Kill(cgid, "Bad fail"); + Kill(h, cgid, "Bad fail"); group.insert(i, md); } } @@ -1439,7 +1439,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM else if (md.size() == 9) field = apf::createFieldOn(mesh, md.name().c_str(), apf::MATRIX); else - Kill(cgid, "Tensor size not accounted for"); + Kill(h, cgid, "Tensor size not accounted for"); double scalar(-123456.0); apf::Vector3 vector3(-123456.0, -123456.0, -123456.0); @@ -1456,7 +1456,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM else if (md.size() == 9) apf::setMatrix(field, elem, 0, matrix3x3); else - Kill(cgid, "Tensor size not accounted for"); + Kill(h, cgid, "Tensor size not accounted for"); } mesh->end(it); @@ -1466,7 +1466,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM { if (cgp_field_read_data(cgid, base, zone, md.sIndex(i), md.fIndex(i), &range_min[0], &range_max[0], meshVals.data())) - Kill(cgid, "Failed cgp_field_read_data"); + Kill(h, cgid, "Failed cgp_field_read_data"); cgsize_t counter = lowest; for (cgsize_t it = 0; it < numToRead; it++) @@ -1498,7 +1498,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM apf::setMatrix(field, elem, 0, matrix3x3); } else - Kill(cgid, "Tensor size not accounted for"); + Kill(h, cgid, "Tensor size not accounted for"); } counter++; } @@ -1509,7 +1509,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM } else { - Kill(cgid, "Don't know how to process this at the moment"); + Kill(h, cgid, "Don't know how to process this at the moment"); } } else if (md.location() == CGNS_ENUMV(CellCenter)) @@ -1535,7 +1535,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM else if (md.size() == 9) field = apf::createField(mesh, md.name().c_str(), apf::MATRIX, apf::getConstant(dim)); else - Kill(cgid, "Tensor size not accounted for"); + Kill(h, cgid, "Tensor size not accounted for"); double scalar(-123456.0); apf::Vector3 vector3(-123456.0, -123456.0, -123456.0); @@ -1552,7 +1552,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM else if (md.size() == 9) apf::setMatrix(field, elem, 0, matrix3x3); else - Kill(cgid, "Tensor size not accounted for"); + Kill(h, cgid, "Tensor size not accounted for"); } mesh->end(it); @@ -1580,7 +1580,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM { if (cgp_field_read_data(cgid, base, zone, md.sIndex(i), md.fIndex(i), &range_min[0], &range_max[0], meshVals.data())) - Kill(cgid, "Failed cgp_field_read_data"); + Kill(h, cgid, "Failed cgp_field_read_data"); for (std::size_t it = 0; it < numToRead; it++) { @@ -1609,7 +1609,7 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM } else { - Kill(cgid, "Tensor size not accounted for"); + Kill(h, cgid, "Tensor size not accounted for"); } } } @@ -1620,19 +1620,19 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM } else { - Kill(cgid, "Don't know how to process this at the moment"); + Kill(h, cgid, "Don't know how to process this at the moment"); } } else { - Kill(cgid, "Don't know how to process this at the moment"); + Kill(h, cgid, "Don't know how to process this at the moment"); } } } } // free up memory - if (PCU_Comm_Initialized()) + if (PCU_Comm_Initialized(h)) cgp_close(cgid); else cg_close(cgid); @@ -1711,10 +1711,10 @@ apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCM return mesh; } // namespace -apf::Mesh2 *DoIt(gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap) +apf::Mesh2 *DoIt(PCU_t h, gmi_model *g, const std::string &fname, apf::CGNSBCMap &cgnsBCMap) { std::vector> meshData; - return DoIt(g, fname, cgnsBCMap, meshData); + return DoIt(h, g, fname, cgnsBCMap, meshData); } } // namespace @@ -1723,10 +1723,10 @@ namespace apf { // caller needs to bring up and pull down mpi/pcu: mpi/pcu is required and assumed. -Mesh2 *loadMdsFromCGNS(gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &meshData) +Mesh2 *loadMdsFromCGNS(PCU_t h, gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap, const std::vector> &meshData) { #ifdef HAVE_CGNS - Mesh2 *m = DoIt(g, fname, cgnsBCMap, meshData); + Mesh2 *m = DoIt(h, g, fname, cgnsBCMap, meshData); return m; #else Mesh2 *m = nullptr; @@ -1738,10 +1738,10 @@ Mesh2 *loadMdsFromCGNS(gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMa } // caller needs to bring up and pull down mpi/pcu: mpi/pcu is required and assumed. -Mesh2 *loadMdsFromCGNS(gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap) +Mesh2 *loadMdsFromCGNS(PCU_t h, gmi_model *g, const char *fname, apf::CGNSBCMap &cgnsBCMap) { #ifdef HAVE_CGNS - Mesh2 *m = DoIt(g, fname, cgnsBCMap); + Mesh2 *m = DoIt(h, g, fname, cgnsBCMap); return m; #else Mesh2 *m = nullptr; diff --git a/mds/mdsGmsh.cc b/mds/mdsGmsh.cc index 4d0ce7875..b6d3f8524 100644 --- a/mds/mdsGmsh.cc +++ b/mds/mdsGmsh.cc @@ -459,17 +459,17 @@ void gmshFindDmg(const char* fnameDmg, const char* filename) } -Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename) +Mesh2* loadMdsFromGmsh(gmi_model* g, const char* filename, pcu::PCU *PCUObj) { - Mesh2* m = makeEmptyMdsMesh(g, 0, false); + Mesh2* m = makeEmptyMdsMesh(g, 0, false, PCUObj); readGmsh(m, filename); return m; } -Mesh2* loadMdsDmgFromGmsh(const char*fnameDmg, const char* filename) +Mesh2* loadMdsDmgFromGmsh(const char*fnameDmg, const char* filename, pcu::PCU *PCUObj) { gmshFindDmg(fnameDmg, filename); // new function that scans $Entities and writes a dmg - Mesh2* m = makeEmptyMdsMesh(gmi_load(fnameDmg), 0, false); + Mesh2* m = makeEmptyMdsMesh(gmi_load(fnameDmg), 0, false, PCUObj); readGmsh(m, filename); return m; } diff --git a/mds/mdsUgrid.cc b/mds/mdsUgrid.cc index ceb8a001b..a96afabe5 100644 --- a/mds/mdsUgrid.cc +++ b/mds/mdsUgrid.cc @@ -661,9 +661,9 @@ namespace { } namespace apf { - Mesh2* loadMdsFromUgrid(gmi_model* g, const char* filename) + Mesh2* loadMdsFromUgrid(gmi_model* g, const char* filename, pcu::PCU *PCUObj) { - Mesh2* m = makeEmptyMdsMesh(g, 0, false); + Mesh2* m = makeEmptyMdsMesh(g, 0, false, PCUObj); header hdr; Reader r; initReader(&r, m, filename); @@ -684,8 +684,8 @@ namespace apf { } void printUgridPtnStats(gmi_model* g, const char* ufile, const char* vtxptn, - const double elmWeights[]) { - Mesh2* m = makeEmptyMdsMesh(g, 0, false); + const double elmWeights[], pcu::PCU *PCUObj) { + Mesh2* m = makeEmptyMdsMesh(g, 0, false, PCUObj); apf::changeMdsDimension(m, 3); printPtnStats(m, ufile, vtxptn, elmWeights); m->destroyNative(); diff --git a/mds/mds_apf.c b/mds/mds_apf.c index 262cd2542..13eb3a1cc 100644 --- a/mds/mds_apf.c +++ b/mds/mds_apf.c @@ -11,7 +11,7 @@ #include "mds_apf.h" #include #include -#include +#include struct mds_apf* mds_apf_create(struct gmi_model* model, int d, mds_id cap[MDS_TYPES]) @@ -148,23 +148,23 @@ int mds_model_id(struct mds_apf* m, struct gmi_ent* model) return gmi_tag(m->user_model, model); } -static void downs_to_copy(struct mds_set* s, +static void downs_to_copy(PCU_t h, struct mds_set* s, struct mds_copy c) { int i; - PCU_COMM_PACK(c.p, c.e); + PCU_COMM_PACK(h, c.p, c.e); for (i = 0; i < s->n; ++i) - PCU_COMM_PACK(c.p, s->e[i]); + PCU_COMM_PACK(h, c.p, s->e[i]); } static void downs_to_copies( - struct mds* m, mds_id e, struct mds_copies* c) + PCU_t h, struct mds* m, mds_id e, struct mds_copies* c) { int i; struct mds_set s; mds_get_adjacent(m, e, mds_dim[mds_type(e)] - 1, &s); for (i = 0; i < c->n; ++i) - downs_to_copy(&s, c->c[i]); + downs_to_copy(h, &s, c->c[i]); } static void change_down(struct mds* m, mds_id e, struct mds_set* s) @@ -219,19 +219,19 @@ static void rotate_set(struct mds_set* in, int r, struct mds_set* out) out->e[i] = in->e[(i + r) % in->n]; } -static int recv_down_copies(struct mds_net* net, struct mds* m) +static int recv_down_copies(PCU_t h, struct mds_net* net, struct mds* m) { mds_id e; struct mds_set s; struct mds_set rs; struct mds_set s2; int i; - int from = PCU_Comm_Sender(); - PCU_COMM_UNPACK(e); + int from = PCU_Comm_Sender(h); + PCU_COMM_UNPACK(h, e); mds_get_adjacent(m, e, mds_dim[mds_type(e)] - 1, &s); rs.n = s.n; for (i = 0; i < s.n; ++i) - PCU_COMM_UNPACK(rs.e[i]); + PCU_COMM_UNPACK(h, rs.e[i]); if (compare_copy_sets(net, &s, from, &rs)) return 0; for (i = -s.n; i < s.n; ++i) { @@ -251,55 +251,57 @@ static int copy_less(struct mds_copy a, struct mds_copy b) return a.e < b.e; } -static int owns_copies(mds_id e, struct mds_copies* c) +static int owns_copies(PCU_t h, mds_id e, struct mds_copies* c) { int i; struct mds_copy mc; mc.e = e; - mc.p = PCU_Comm_Self(); + mc.p = PCU_Comm_Self(h); for (i = 0; i < c->n; ++i) if (copy_less(c->c[i], mc)) return 0; return 1; } -static int align_copies(struct mds_net* net, struct mds* m) +static int align_copies(PCU_t h, struct mds_net* net, struct mds* m) { int d; mds_id e; struct mds_copies* c; int did_change = 0; for (d = 1; d < m->d; ++d){ - PCU_Comm_Begin(); + PCU_Comm_Begin(h); for (e = mds_begin(m, d); e != MDS_NONE; e = mds_next(m, e)) { c = mds_get_copies(net, e); if (!c) continue; - if (owns_copies(e, c)) - downs_to_copies(m, e, c); + if (owns_copies(h, e, c)) + downs_to_copies(h, m, e, c); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) - if (recv_down_copies(net, m)) + PCU_Comm_Send(h); + while (PCU_Comm_Receive(h)) + if (recv_down_copies(h, net, m)) did_change = 1; } - return PCU_Or(did_change); + return PCU_Or(h, did_change); } -int mds_align_matches(struct mds_apf* m) + +int mds_align_matches(PCU_t h, struct mds_apf* m) { - return align_copies(&m->matches, &m->mds); + return align_copies(h, &m->matches, &m->mds); } // seol -int mds_align_ghosts(struct mds_apf* m) +int mds_align_ghosts(PCU_t h, struct mds_apf* m) { - return align_copies(&m->ghosts, &m->mds); + return align_copies(h, &m->ghosts, &m->mds); } -int mds_align_remotes(struct mds_apf* m) + +int mds_align_remotes(PCU_t h, struct mds_apf* m) { - return align_copies(&m->remotes, &m->mds); + return align_copies(h, &m->remotes, &m->mds); } void mds_update_model_for_entity(struct mds_apf* m, mds_id e, diff --git a/mds/mds_apf.h b/mds/mds_apf.h index fb51e2ff2..65fff23d9 100644 --- a/mds/mds_apf.h +++ b/mds/mds_apf.h @@ -26,6 +26,8 @@ struct pcu_file; struct gmi_model; struct gmi_ent; +typedef struct PCU_t PCU_t; + struct mds_apf { struct mds mds; struct mds_tags tags; @@ -55,24 +57,24 @@ void* mds_get_part(struct mds_apf* m, mds_id e); void mds_set_part(struct mds_apf* m, mds_id e, void* p); struct mds_tag* mds_number_verts_bfs(struct mds_apf* m); -struct mds_apf* mds_reorder(struct mds_apf* m, int ignore_peers, +struct mds_apf* mds_reorder(PCU_t h, struct mds_apf* m, int ignore_peers, struct mds_tag* vert_numbers); struct gmi_ent* mds_find_model(struct mds_apf* m, int dim, int id); int mds_model_dim(struct mds_apf* m, struct gmi_ent* model); int mds_model_id(struct mds_apf* m, struct gmi_ent* model); -struct mds_apf* mds_read_smb(struct gmi_model* model, const char* pathname, +struct mds_apf* mds_read_smb(PCU_t h, struct gmi_model* model, const char* pathname, int ignore_peers, void* apf_mesh); -struct mds_apf* mds_write_smb(struct mds_apf* m, const char* pathname, +struct mds_apf* mds_write_smb(PCU_t h, struct mds_apf* m, const char* pathname, int ignore_peers, void* apf_mesh); void mds_verify(struct mds_apf* m); void mds_verify_residence(struct mds_apf* m, mds_id e); -int mds_align_matches(struct mds_apf* m); -int mds_align_ghosts(struct mds_apf* m); -int mds_align_remotes(struct mds_apf* m); +int mds_align_matches(PCU_t h, struct mds_apf* m); +int mds_align_ghosts(PCU_t h, struct mds_apf* m); +int mds_align_remotes(PCU_t h, struct mds_apf* m); void mds_derive_model(struct mds_apf* m); void mds_update_model_for_entity(struct mds_apf* m, mds_id e, diff --git a/mds/mds_net.c b/mds/mds_net.c index 2782ab368..ab59b86bb 100644 --- a/mds/mds_net.c +++ b/mds/mds_net.c @@ -9,7 +9,7 @@ *******************************************************************************/ #include "mds_net.h" -#include +#include #include #include #include @@ -159,8 +159,8 @@ static void note_peer(struct mds_links* ln, int p) ++(ln->n[i]); } -static void for_type_net(struct mds_net* net, struct mds* m, - int t, void (*f)(mds_id i, struct mds_copy c, void* u), void* u) +static void for_type_net(PCU_t h, struct mds_net* net, struct mds* m, + int t, void (*f)(PCU_t h, mds_id i, struct mds_copy c, void* u), void* u) { mds_id i; int j; @@ -170,14 +170,14 @@ static void for_type_net(struct mds_net* net, struct mds* m, if (!cs) continue; for (j = 0; j < cs->n; ++j) - f(i, cs->c[j], u); + f(h, i, cs->c[j], u); } } -static void note_remote_link(mds_id i, struct mds_copy c, void* u) +static void note_remote_link(PCU_t h, mds_id i, struct mds_copy c, void* u) { (void)i; - if (c.p != PCU_Comm_Self()) + if (c.p != PCU_Comm_Self(h)) note_peer(u, c.p); } @@ -189,11 +189,11 @@ static void alloc_links(struct mds_links* ln) ln->l[i] = malloc(ln->n[i] * sizeof(unsigned)); } -static void take_remote_link(mds_id i, struct mds_copy c, void* u) +static void take_remote_link(PCU_t h, mds_id i, struct mds_copy c, void* u) { struct mds_links* ln = u; int pi; - if (PCU_Comm_Self() < c.p) { + if (PCU_Comm_Self(h) < c.p) { pi = find_peer(ln, c.p); ln->l[pi][ln->n[pi]] = i; /* use n as position keeper */ @@ -201,68 +201,68 @@ static void take_remote_link(mds_id i, struct mds_copy c, void* u) } } -static void send_remote_link(mds_id i, struct mds_copy c, void* u) +static void send_remote_link(PCU_t h, mds_id i, struct mds_copy c, void* u) { (void)i; (void)u; - if (PCU_Comm_Self() < c.p) - PCU_COMM_PACK(c.p, c.e); + if (PCU_Comm_Self(h) < c.p) + PCU_COMM_PACK(h, c.p, c.e); } -static void recv_links(struct mds_links* ln) +static void recv_links(PCU_t h, struct mds_links* ln) { int from; mds_id* tmp; int pi; unsigned i; - from = PCU_Comm_Sender(); + from = PCU_Comm_Sender(h); pi = find_peer(ln, from); - tmp = PCU_Comm_Extract(ln->n[pi] * sizeof(mds_id)); + tmp = PCU_Comm_Extract(h, ln->n[pi] * sizeof(mds_id)); for (i = 0; i < ln->n[pi]; ++i) ln->l[pi][i] = mds_index(tmp[i]); } -void mds_get_type_links(struct mds_net* net, struct mds* m, +void mds_get_type_links(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { unsigned i; /* count remote links in np, p and n */ - for_type_net(net, m, t, note_remote_link, ln); + for_type_net(h, net, m, t, note_remote_link, ln); alloc_links(ln); for (i = 0; i < ln->np; ++i) - if (((unsigned)PCU_Comm_Self()) < ln->p[i]) + if (((unsigned)PCU_Comm_Self(h)) < ln->p[i]) /* zero n's for behavior in take_copy */ ln->n[i] = 0; /* record indices in local order for owned boundaries */ - for_type_net(net, m, t, take_remote_link, ln); - PCU_Comm_Begin(); + for_type_net(h, net, m, t, take_remote_link, ln); + PCU_Comm_Begin(h); /* send indices in local order for owned boundaries */ - for_type_net(net, m, t, send_remote_link, NULL); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) + for_type_net(h, net, m, t, send_remote_link, NULL); + PCU_Comm_Send(h); + while (PCU_Comm_Listen(h)) /* recv indices in remote order for non-owned boundaries */ - recv_links(ln); + recv_links(h, ln); } -void mds_set_type_links(struct mds_net* net, struct mds* m, +void mds_set_type_links(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { unsigned i; unsigned j; unsigned* in; struct mds_copy c; - PCU_Comm_Begin(); + PCU_Comm_Begin(h); for (i = 0; i < ln->np; ++i) { PCU_ALWAYS_ASSERT(ln->l); for (j = 0; j < ln->n[i]; ++j) - PCU_COMM_PACK(ln->p[i], ln->l[i][j]); + PCU_COMM_PACK(h, ln->p[i], ln->l[i][j]); } - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { - c.p = PCU_Comm_Sender(); - PCU_ALWAYS_ASSERT(c.p != PCU_Comm_Self()); + PCU_Comm_Send(h); + while (PCU_Comm_Listen(h)) { + c.p = PCU_Comm_Sender(h); + PCU_ALWAYS_ASSERT(c.p != PCU_Comm_Self(h)); i = find_peer(ln, c.p); - in = PCU_Comm_Extract(ln->n[i] * sizeof(unsigned)); + in = PCU_Comm_Extract(h, ln->n[i] * sizeof(unsigned)); for (j = 0; j < ln->n[i]; ++j) { c.e = mds_identify(t, in[j]); mds_add_copy(net, m, mds_identify(t, ln->l[i][j]), c); @@ -289,23 +289,23 @@ int mds_net_empty(struct mds_net* net) return 1; } -static void note_local_link(mds_id i, struct mds_copy c, void* u) +static void note_local_link(PCU_t h, mds_id i, struct mds_copy c, void* u) { - if (c.p == PCU_Comm_Self()) { + if (c.p == PCU_Comm_Self(h)) { if (i < mds_index(c.e)) - note_peer(u, PCU_Comm_Self()); + note_peer(u, PCU_Comm_Self(h)); else /* hack id to store self-receivers */ - note_peer(u, PCU_Comm_Peers()); + note_peer(u, PCU_Comm_Peers(h)); } } -static void take_local_link(mds_id i, struct mds_copy c, void* u) +static void take_local_link(PCU_t h, mds_id i, struct mds_copy c, void* u) { struct mds_links* ln = u; - int self = find_peer(ln, PCU_Comm_Self()); - int other = find_peer(ln, PCU_Comm_Peers()); + int self = find_peer(ln, PCU_Comm_Self(h)); + int other = find_peer(ln, PCU_Comm_Peers(h)); mds_id j = mds_index(c.e); - if ((PCU_Comm_Self() == c.p) && (i < j)) { + if ((PCU_Comm_Self(h) == c.p) && (i < j)) { ln->l[self][ln->n[self]] = i; ln->l[other][ln->n[other]] = j; /* use ns as (redundant) position keepers */ @@ -314,35 +314,35 @@ static void take_local_link(mds_id i, struct mds_copy c, void* u) } } -void mds_get_local_matches(struct mds_net* net, struct mds* m, +void mds_get_local_matches(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { int self, other; - for_type_net(net, m, t, note_local_link, ln); - self = find_peer(ln, PCU_Comm_Self()); + for_type_net(h, net, m, t, note_local_link, ln); + self = find_peer(ln, PCU_Comm_Self(h)); if (self == -1) return; - other = find_peer(ln, PCU_Comm_Peers()); + other = find_peer(ln, PCU_Comm_Peers(h)); PCU_ALWAYS_ASSERT(ln->n[self] == ln->n[other]); ln->l[self] = malloc(ln->n[self] * sizeof(unsigned)); ln->l[other] = malloc(ln->n[other] * sizeof(unsigned)); ln->n[self] = 0; ln->n[other] = 0; - for_type_net(net, m, t, take_local_link, ln); + for_type_net(h, net, m, t, take_local_link, ln); } -void mds_set_local_matches(struct mds_net* net, struct mds* m, +void mds_set_local_matches(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln) { int self, other; unsigned i; mds_id a, b; struct mds_copy c; - c.p = PCU_Comm_Self(); - self = find_peer(ln, PCU_Comm_Self()); + c.p = PCU_Comm_Self(h); + self = find_peer(ln, PCU_Comm_Self(h)); if (self == -1) return; - other = find_peer(ln, PCU_Comm_Peers()); + other = find_peer(ln, PCU_Comm_Peers(h)); PCU_ALWAYS_ASSERT(ln->n != 0); PCU_ALWAYS_ASSERT(ln->n[self] == ln->n[other]); for (i = 0; i < ln->n[self]; ++i) { @@ -356,13 +356,13 @@ void mds_set_local_matches(struct mds_net* net, struct mds* m, } } -void mds_free_local_links(struct mds_links* ln) +void mds_free_local_links(PCU_t h, struct mds_links* ln) { int self, other; - self = find_peer(ln, PCU_Comm_Self()); + self = find_peer(ln, PCU_Comm_Self(h)); if (self == -1) return; - other = find_peer(ln, PCU_Comm_Peers()); + other = find_peer(ln, PCU_Comm_Peers(h)); PCU_ALWAYS_ASSERT(ln->n != 0); ln->n[self] = ln->n[other] = 0; PCU_ALWAYS_ASSERT(ln->l != 0); diff --git a/mds/mds_net.h b/mds/mds_net.h index 69dec52b9..8cc7de09d 100644 --- a/mds/mds_net.h +++ b/mds/mds_net.h @@ -13,6 +13,8 @@ #include "mds.h" +typedef struct PCU_t PCU_t; + struct mds_copy { mds_id e; int p; @@ -50,18 +52,18 @@ void mds_grow_net( void mds_add_copy(struct mds_net* net, struct mds* m, mds_id e, struct mds_copy c); -void mds_get_type_links(struct mds_net* net, struct mds* m, +void mds_get_type_links(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln); -void mds_set_type_links(struct mds_net* net, struct mds* m, +void mds_set_type_links(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln); void mds_free_links(struct mds_links* ln); int mds_net_empty(struct mds_net* net); -void mds_get_local_matches(struct mds_net* net, struct mds* m, +void mds_get_local_matches(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln); -void mds_set_local_matches(struct mds_net* net, struct mds* m, +void mds_set_local_matches(PCU_t h, struct mds_net* net, struct mds* m, int t, struct mds_links* ln); -void mds_free_local_links(struct mds_links* ln); +void mds_free_local_links(PCU_t h, struct mds_links* ln); #endif diff --git a/mds/mds_order.c b/mds/mds_order.c index d8ac16921..ffc41b161 100644 --- a/mds/mds_order.c +++ b/mds/mds_order.c @@ -13,7 +13,7 @@ #include #include #include -#include +#include struct queue { mds_id* e; @@ -183,7 +183,7 @@ static mds_id lookup(struct mds_tag* tag, mds_id old) } /* see apf/apfConvert.cc apf::Converter::createRemotes */ -static void rebuild_net(struct mds_net* net, +static void rebuild_net(PCU_t h, struct mds_net* net, struct mds* m, struct mds_net* net2, struct mds* m2, @@ -197,7 +197,7 @@ static void rebuild_net(struct mds_net* net, struct mds_copies* cs; struct mds_copy c; int i; - PCU_Comm_Begin(); + PCU_Comm_Begin(h); for (d = 0; d <= m->d; ++d) for (e = mds_begin(m, d); e != MDS_NONE; e = mds_next(m, e)) { cs = mds_get_copies(net, e); @@ -206,16 +206,16 @@ static void rebuild_net(struct mds_net* net, ne = lookup(new_of, e); for (i = 0; i < cs->n; ++i) { ce = cs->c[i].e; - PCU_COMM_PACK(cs->c[i].p, ce); - PCU_COMM_PACK(cs->c[i].p, ne); + PCU_COMM_PACK(h, cs->c[i].p, ce); + PCU_COMM_PACK(h, cs->c[i].p, ne); } } - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { - c.p = PCU_Comm_Sender(); - while (!PCU_Comm_Unpacked()) { - PCU_COMM_UNPACK(ce); - PCU_COMM_UNPACK(ne); + PCU_Comm_Send(h); + while (PCU_Comm_Listen(h)) { + c.p = PCU_Comm_Sender(h); + while (!PCU_Comm_Unpacked(h)) { + PCU_COMM_UNPACK(h, ce); + PCU_COMM_UNPACK(h, ne); c.e = ne; nce = lookup(new_of, ce); mds_add_copy(net2, m2, nce, c); @@ -380,6 +380,7 @@ static void rebuild_parts( } static struct mds_apf* rebuild( + PCU_t h, struct mds_apf* m, struct mds_tag* new_of, int ignore_peers) @@ -394,10 +395,10 @@ static struct mds_apf* rebuild( rebuild_coords(m, m2, old_of); rebuild_parts(m, m2, old_of); if (!ignore_peers) { - rebuild_net(&m->remotes, &m->mds, + rebuild_net(h, &m->remotes, &m->mds, &m2->remotes, &m2->mds, new_of); - rebuild_net(&m->matches, &m->mds, + rebuild_net(h, &m->matches, &m->mds, &m2->matches, &m2->mds, new_of); } @@ -405,14 +406,14 @@ static struct mds_apf* rebuild( return m2; } -struct mds_apf* mds_reorder(struct mds_apf* m, int ignore_peers, +struct mds_apf* mds_reorder(PCU_t h, struct mds_apf* m, int ignore_peers, struct mds_tag* vert_numbers) { struct mds_tag* tag; struct mds_apf* m2; tag = vert_numbers; number_other_ents(m, tag); - m2 = rebuild(m, tag, ignore_peers); + m2 = rebuild(h, m, tag, ignore_peers); mds_apf_destroy(m); return m2; } diff --git a/mds/mds_smb.c b/mds/mds_smb.c index f0590d679..882ebcc94 100644 --- a/mds/mds_smb.c +++ b/mds/mds_smb.c @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -51,6 +51,8 @@ enum { #define MAX_PEERS (10*1000) #define MAX_TAGS (100) +typedef struct PCU_t PCU_t; + static int smb2mds(int smb_type) { int const table[SMB_TYPES] = @@ -116,7 +118,7 @@ static void write_links(struct pcu_file* f, struct mds_links* l) pcu_write_unsigneds(f, l->l[i], l->n[i]); } -static void read_header(struct pcu_file* f, unsigned* version, unsigned* dim, +static void read_header(PCU_t h, struct pcu_file* f, unsigned* version, unsigned* dim, int ignore_peers) { unsigned magic, np; @@ -126,12 +128,12 @@ static void read_header(struct pcu_file* f, unsigned* version, unsigned* dim, PCU_READ_UNSIGNED(f, *dim); PCU_READ_UNSIGNED(f, np); if (*version >= 1 && (!ignore_peers)) - if (np != (unsigned)PCU_Comm_Peers()) + if (np != (unsigned)PCU_Comm_Peers(h)) reel_fail("To whom it may concern\n" "the # of mesh partitions != the # of MPI ranks"); } -static void write_header(struct pcu_file* f, unsigned dim, +static void write_header(PCU_t h, struct pcu_file* f, unsigned dim, int ignore_peers) { unsigned magic = 0; @@ -143,7 +145,7 @@ static void write_header(struct pcu_file* f, unsigned dim, if (ignore_peers) np = 1; else - np = (unsigned)PCU_Comm_Peers(); + np = (unsigned)PCU_Comm_Peers(h); PCU_WRITE_UNSIGNED(f, np); } @@ -210,22 +212,22 @@ static void write_conn(struct pcu_file* f, struct mds_apf* m) } } -static void read_remotes(struct pcu_file* f, struct mds_apf* m, +static void read_remotes(PCU_t h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { struct mds_links ln = MDS_LINKS_INIT; read_links(f, &ln); if (!ignore_peers) - mds_set_type_links(&m->remotes, &m->mds, MDS_VERTEX, &ln); + mds_set_type_links(h, &m->remotes, &m->mds, MDS_VERTEX, &ln); mds_free_links(&ln); } -static void write_remotes(struct pcu_file* f, struct mds_apf* m, +static void write_remotes(PCU_t h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { struct mds_links ln = MDS_LINKS_INIT; if (!ignore_peers) - mds_get_type_links(&m->remotes, &m->mds, MDS_VERTEX, &ln); + mds_get_type_links(h, &m->remotes, &m->mds, MDS_VERTEX, &ln); write_links(f, &ln); mds_free_links(&ln); } @@ -508,56 +510,56 @@ static void write_tags(struct pcu_file* f, struct mds_apf* m) free(sizes); } -static void read_type_matches(struct pcu_file* f, struct mds_apf* m, int t, +static void read_type_matches(PCU_t h, struct pcu_file* f, struct mds_apf* m, int t, int ignore_peers) { struct mds_links ln = MDS_LINKS_INIT; read_links(f, &ln); if (!ignore_peers) - mds_set_local_matches(&m->matches, &m->mds, t, &ln); - mds_free_local_links(&ln); + mds_set_local_matches(h, &m->matches, &m->mds, t, &ln); + mds_free_local_links(h, &ln); if (!ignore_peers) - mds_set_type_links(&m->matches, &m->mds, t, &ln); + mds_set_type_links(h, &m->matches, &m->mds, t, &ln); mds_free_links(&ln); } -static void write_type_matches(struct pcu_file* f, struct mds_apf* m, int t, +static void write_type_matches(PCU_t h, struct pcu_file* f, struct mds_apf* m, int t, int ignore_peers) { struct mds_links ln = MDS_LINKS_INIT; if (!ignore_peers) { - mds_get_type_links(&m->matches, &m->mds, t, &ln); - mds_get_local_matches(&m->matches, &m->mds, t, &ln); + mds_get_type_links(h, &m->matches, &m->mds, t, &ln); + mds_get_local_matches(h, &m->matches, &m->mds, t, &ln); } write_links(f, &ln); mds_free_links(&ln); } -static void read_matches_old(struct pcu_file* f, struct mds_apf* m, +static void read_matches_old(PCU_t h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { int t; for (t = 0; t < MDS_HEXAHEDRON; ++t) - read_type_matches(f, m, t, ignore_peers); + read_type_matches(h, f, m, t, ignore_peers); } -static void read_matches_new(struct pcu_file* f, struct mds_apf* m, +static void read_matches_new(PCU_t h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { int t; for (t = 0; t < SMB_TYPES; ++t) - read_type_matches(f, m, smb2mds(t), ignore_peers); + read_type_matches(h, f, m, smb2mds(t), ignore_peers); } -static void write_matches(struct pcu_file* f, struct mds_apf* m, +static void write_matches(PCU_t h, struct pcu_file* f, struct mds_apf* m, int ignore_peers) { int t; for (t = 0; t < SMB_TYPES; ++t) - write_type_matches(f, m, smb2mds(t), ignore_peers); + write_type_matches(h, f, m, smb2mds(t), ignore_peers); } -static struct mds_apf* read_smb(struct gmi_model* model, const char* filename, +static struct mds_apf* read_smb(PCU_t h, struct gmi_model* model, const char* filename, int zip, int ignore_peers, void* apf_mesh) { struct mds_apf* m; @@ -569,9 +571,9 @@ static struct mds_apf* read_smb(struct gmi_model* model, const char* filename, int i; unsigned tmp; unsigned pi, pj; - f = pcu_fopen(filename, 0, zip); + f = pcu_fopen(h, filename, 0, zip); PCU_ALWAYS_ASSERT(f); - read_header(f, &version, &dim, ignore_peers); + read_header(h, f, &version, &dim, ignore_peers); pcu_read_unsigneds(f, n, SMB_TYPES); for (i = 0; i < MDS_TYPES; ++i) { tmp = n[mds2smb(i)]; @@ -590,13 +592,13 @@ static struct mds_apf* read_smb(struct gmi_model* model, const char* filename, for (pj = 0; pj < 2; ++pj) m->param[pi][pj] = 0.0; } } - read_remotes(f, m, ignore_peers); + read_remotes(h, f, m, ignore_peers); read_class(f, m); read_tags(f, m); if (version >= 4) - read_matches_new(f, m, ignore_peers); + read_matches_new(h, f, m, ignore_peers); else if (version >= 3) - read_matches_old(f, m, ignore_peers); + read_matches_old(h, f, m, ignore_peers); if (version >= 5) mds_read_smb_meta(f, m, apf_mesh); pcu_fclose(f); @@ -612,24 +614,24 @@ static void write_coords(struct pcu_file* f, struct mds_apf* m) pcu_write_doubles(f, &m->param[0][0], count); } -static void write_smb(struct mds_apf* m, const char* filename, +static void write_smb(PCU_t h, struct mds_apf* m, const char* filename, int zip, int ignore_peers, void* apf_mesh) { struct pcu_file* f; unsigned n[SMB_TYPES] = {0}; int i; - f = pcu_fopen(filename, 1, zip); + f = pcu_fopen(h, filename, 1, zip); PCU_ALWAYS_ASSERT(f); - write_header(f, m->mds.d, ignore_peers); + write_header(h, f, m->mds.d, ignore_peers); for (i = 0; i < MDS_TYPES; ++i) n[mds2smb(i)] = m->mds.end[i]; pcu_write_unsigneds(f, n, SMB_TYPES); write_conn(f, m); write_coords(f, m); - write_remotes(f, m, ignore_peers); + write_remotes(h, f, m, ignore_peers); write_class(f, m); write_tags(f, m); - write_matches(f, m, ignore_peers); + write_matches(h, f, m, ignore_peers); mds_write_smb_meta(f, apf_mesh); pcu_fclose(f); } @@ -686,7 +688,7 @@ static void safe_mkdir(const char* path, mode_t mode) reel_fail("MDS: could not create directory \"%s\"\n", path); } -static char* handle_path(const char* in, int is_write, int* zip, +static char* handle_path(PCU_t h, const char* in, int is_write, int* zip, int ignore_peers) { static const char* zippre = "bz2:"; @@ -694,7 +696,7 @@ static char* handle_path(const char* in, int is_write, int* zip, size_t bufsize; char* path; mode_t const dir_perm = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH; - int self = PCU_Comm_Self(); + int self = PCU_Comm_Self(h); bufsize = strlen(in) + 256; path = malloc(bufsize); strcpy(path, in); @@ -710,14 +712,14 @@ static char* handle_path(const char* in, int is_write, int* zip, if (is_write) { if (!self) safe_mkdir(path, dir_perm); - PCU_Barrier(); + PCU_Barrier(h); } - if (PCU_Comm_Peers() > SMB_FANOUT) { + if (PCU_Comm_Peers(h) > SMB_FANOUT) { append(path, bufsize, "%d/", self / SMB_FANOUT); if (is_write) { if (self % SMB_FANOUT == 0) safe_mkdir(path, dir_perm); - PCU_Barrier(); + PCU_Barrier(h); } } } else if (ends_with(path, smbext)) { @@ -729,14 +731,14 @@ static char* handle_path(const char* in, int is_write, int* zip, return path; } -struct mds_apf* mds_read_smb(struct gmi_model* model, const char* pathname, +struct mds_apf* mds_read_smb(PCU_t h, struct gmi_model* model, const char* pathname, int ignore_peers, void* apf_mesh) { char* filename; int zip; struct mds_apf* m; - filename = handle_path(pathname, 0, &zip, ignore_peers); - m = read_smb(model, filename, zip, ignore_peers, apf_mesh); + filename = handle_path(h, pathname, 0, &zip, ignore_peers); + m = read_smb(h, model, filename, zip, ignore_peers, apf_mesh); free(filename); return m; } @@ -750,22 +752,22 @@ static int is_compact(struct mds_apf* m) return 1; } -struct mds_apf* mds_write_smb(struct mds_apf* m, const char* pathname, +struct mds_apf* mds_write_smb(PCU_t h, struct mds_apf* m, const char* pathname, int ignore_peers, void* apf_mesh) { const char* reorderWarning ="MDS: reordering before writing smb files\n"; char* filename; int zip; if (ignore_peers && (!is_compact(m))) { - if(!PCU_Comm_Self()) lion_eprint(1, "%s", reorderWarning); - m = mds_reorder(m, 1, mds_number_verts_bfs(m)); + if(!PCU_Comm_Self(h)) lion_eprint(1, "%s", reorderWarning); + m = mds_reorder(h, m, 1, mds_number_verts_bfs(m)); } - if ((!ignore_peers) && PCU_Or(!is_compact(m))) { - if(!PCU_Comm_Self()) lion_eprint(1, "%s", reorderWarning); - m = mds_reorder(m, 0, mds_number_verts_bfs(m)); + if ((!ignore_peers) && PCU_Or(h, !is_compact(m))) { + if(!PCU_Comm_Self(h)) lion_eprint(1, "%s", reorderWarning); + m = mds_reorder(h, m, 0, mds_number_verts_bfs(m)); } - filename = handle_path(pathname, 1, &zip, ignore_peers); - write_smb(m, filename, zip, ignore_peers, apf_mesh); + filename = handle_path(h, pathname, 1, &zip, ignore_peers); + write_smb(h, m, filename, zip, ignore_peers, apf_mesh); free(filename); return m; } diff --git a/omega_h/apfOmega_h.cc b/omega_h/apfOmega_h.cc index f854b7cc3..663831af8 100644 --- a/omega_h/apfOmega_h.cc +++ b/omega_h/apfOmega_h.cc @@ -7,7 +7,6 @@ #include #include #include -#include #include #include @@ -125,7 +124,7 @@ static void field_to_osh(osh::Mesh* om, apf::Field* f) { ) { ent_dim = dim; } else { - if (!PCU_Comm_Self()) { + if (!am->getPCU()->Self()) { lion_oprint(1,"not copying field %s to Omega_h\n",name.c_str()); } return; @@ -181,7 +180,7 @@ static void field_from_osh(apf::Mesh* am, osh::Tag const* tag, if (ent_dim == 0) shape = apf::getLagrange(1); else if (ent_dim == dim) shape = apf::getIPFitShape(dim, 1); else { - if (!PCU_Comm_Self()) { + if (!am->getPCU()->Self()) { lion_oprint(1,"not copying field %s to Omega_h\n",name.c_str()); } return; @@ -293,7 +292,7 @@ static void globals_to_osh( } void to_omega_h(osh::Mesh* om, apf::Mesh* am) { - auto comm_mpi = PCU_Get_Comm(); + auto comm_mpi = am->getPCU()->GetMPIComm(); decltype(comm_mpi) comm_impl; MPI_Comm_dup(comm_mpi, &comm_impl); auto comm_osh = osh::CommPtr(new osh::Comm(om->library(), comm_impl)); @@ -385,56 +384,56 @@ static void owners_from_osh( am->setIntTag(ents[i], own_tag, &tmp); } } - PCU_Comm_Begin(); + am->getPCU()->Begin(); for (int i = 0; i < om->nents(ent_dim); ++i) { osh::LO tmp = own_ids[i]; - PCU_COMM_PACK(own_ranks[i], tmp); - PCU_COMM_PACK(own_ranks[i], ents[i]); + am->getPCU()->Pack(own_ranks[i], tmp); + am->getPCU()->Pack(own_ranks[i], ents[i]); } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + am->getPCU()->Send(); + while (am->getPCU()->Receive()) { int own_id; - PCU_COMM_UNPACK(own_id); + am->getPCU()->Unpack(own_id); apf::MeshEntity* r; - PCU_COMM_UNPACK(r); - int from = PCU_Comm_Sender(); - if (from == PCU_Comm_Self()) { + am->getPCU()->Unpack(r); + int from = am->getPCU()->Sender(); + if (from == am->getPCU()->Self()) { assert((int)from == own_ranks[own_id]); continue; } apf::MeshEntity* owner = ents[own_id]; am->addRemote(owner, from, r); } - PCU_Comm_Begin(); + am->getPCU()->Begin(); for (int i = 0; i < om->nents(ent_dim); ++i) - if (own_ranks[i] == (int)(PCU_Comm_Self())) { + if (own_ranks[i] == (int)(am->getPCU()->Self())) { apf::Copies remotes; am->getRemotes(ents[i], remotes); int ncopies = remotes.size(); - int self = PCU_Comm_Self(); + int self = am->getPCU()->Self(); APF_ITERATE(apf::Copies, remotes, it) { - PCU_COMM_PACK(it->first, it->second); - PCU_COMM_PACK(it->first, ncopies); - PCU_COMM_PACK(it->first, self); - PCU_COMM_PACK(it->first, ents[i]); + am->getPCU()->Pack(it->first, it->second); + am->getPCU()->Pack(it->first, ncopies); + am->getPCU()->Pack(it->first, self); + am->getPCU()->Pack(it->first, ents[i]); APF_ITERATE(apf::Copies, remotes, it2) if (it2->first != it->first) { - PCU_COMM_PACK(it->first, it2->first); - PCU_COMM_PACK(it->first, it2->second); + am->getPCU()->Pack(it->first, it2->first); + am->getPCU()->Pack(it->first, it2->second); } } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + am->getPCU()->Send(); + while (am->getPCU()->Receive()) { apf::MeshEntity* e; - PCU_COMM_UNPACK(e); + am->getPCU()->Unpack(e); int ncopies; - PCU_COMM_UNPACK(ncopies); + am->getPCU()->Unpack(ncopies); for (int i = 0; i < ncopies; ++i) { int p; - PCU_COMM_UNPACK(p); + am->getPCU()->Unpack(p); apf::MeshEntity* r; - PCU_COMM_UNPACK(r); + am->getPCU()->Unpack(r); am->addRemote(e, p, r); } } diff --git a/parma/diffMC/maximalIndependentSet/mersenne_twister.cc b/parma/diffMC/maximalIndependentSet/mersenne_twister.cc index 613e08ff1..8938e2cff 100644 --- a/parma/diffMC/maximalIndependentSet/mersenne_twister.cc +++ b/parma/diffMC/maximalIndependentSet/mersenne_twister.cc @@ -1,5 +1,4 @@ #include "mersenne_twister.h" -#include "PCU.h" #include #include #include @@ -34,8 +33,8 @@ static unsigned long mt[N]; /* the array for the state vector */ static int mti = N + 1; /* mti == N + 1 means mt is not initialized */ namespace { - void fail(const char* msg) { - if( ! PCU_Comm_Self() ) + void fail(const char* msg, pcu::PCU *PCUObj) { + if( ! PCUObj->Self() ) lion_eprint(1, "%s", msg); exit(EXIT_FAILURE); } @@ -53,14 +52,14 @@ void mersenne_twister_seed(unsigned seed) mt[mti] = (6909 * mt[mti - 1]) & 0xFFFFFFFF; } -unsigned mersenne_twister(void) +unsigned mersenne_twister(pcu::PCU *PCUObj) { static unsigned long const mag01[2] = {0x0, MATRIX_A}; /* mag01[x] = x * MATRIX_A fox x = {0,1} */ unsigned long y; int kk; if (mti == N + 1) - fail("mersenne twister was not seeded before use\n"); + fail("mersenne twister was not seeded before use\n", PCUObj); if (mti == N) { /* generate N words at one time */ for (kk = 0; kk < N - M; ++kk) { y = (mt[kk] & UPPER_MASK) | (mt[kk + 1] & LOWER_MASK); diff --git a/parma/diffMC/maximalIndependentSet/mersenne_twister.h b/parma/diffMC/maximalIndependentSet/mersenne_twister.h index 235c4ef62..a6dcbb365 100644 --- a/parma/diffMC/maximalIndependentSet/mersenne_twister.h +++ b/parma/diffMC/maximalIndependentSet/mersenne_twister.h @@ -1,7 +1,9 @@ #ifndef MERSENNE_TWISTER_H #define MERSENNE_TWISTER_H +#include + void mersenne_twister_seed(unsigned seed); -unsigned mersenne_twister(void); +unsigned mersenne_twister(pcu::PCU *PCUObj); #endif diff --git a/parma/diffMC/maximalIndependentSet/mis.h b/parma/diffMC/maximalIndependentSet/mis.h index 2c760435b..291d7c6a4 100644 --- a/parma/diffMC/maximalIndependentSet/mis.h +++ b/parma/diffMC/maximalIndependentSet/mis.h @@ -52,7 +52,7 @@ namespace misLuby { * @param randNumsPredefined (In) 0: compute random numbers, 1:uses defined random numbers * @return 1 if local part is in mis, 0 o.w. */ -int mis(misLuby::partInfo& part, +int mis(misLuby::partInfo& part, pcu::PCU *PCUObj, bool randNumsPredefined = false, bool isNeighbors = false); diff --git a/parma/diffMC/maximalIndependentSet/misLuby.cc b/parma/diffMC/maximalIndependentSet/misLuby.cc index b2b2fbf5e..ecabc915a 100644 --- a/parma/diffMC/maximalIndependentSet/misLuby.cc +++ b/parma/diffMC/maximalIndependentSet/misLuby.cc @@ -35,12 +35,12 @@ using std::set; using namespace misLuby; namespace { - int generateRandomNumber() { - return (int) mersenne_twister(); + int generateRandomNumber(pcu::PCU *PCUObj) { + return (int) mersenne_twister(PCUObj); } - void setRandomNum(partInfo& part) { - part.randNum = generateRandomNumber(); + void setRandomNum(partInfo& part, pcu::PCU *PCUObj) { + part.randNum = generateRandomNumber(PCUObj); // don't select self nets until all other nets are selected if ( 1 == part.net.size() ) part.randNum = std::numeric_limits::max(); @@ -52,31 +52,31 @@ namespace { p.netAdjParts.erase(*nodeItr); } - int sendAdjNetsToNeighbors(partInfo& part, vector& nbNet) { - PCU_Comm_Begin(); + int sendAdjNetsToNeighbors(partInfo& part, vector& nbNet, pcu::PCU *PCUObj) { + PCUObj->Begin(); MIS_ITERATE(vector, part.adjPartIds, adjPartIdItr) { const int destRank = *adjPartIdItr; MIS_ITERATE(vector, nbNet, nbItr) { if (nbItr->partId != *adjPartIdItr) { //pack destination part Id - PCU_COMM_PACK(destRank, *adjPartIdItr); + PCUObj->Pack(destRank, *adjPartIdItr); //adjacent part Id - PCU_COMM_PACK(destRank, nbItr->partId); + PCUObj->Pack(destRank, nbItr->partId); //adjacent part's random number - PCU_COMM_PACK(destRank, nbItr->randNum); + PCUObj->Pack(destRank, nbItr->randNum); //net size size_t n = nbItr->net.size(); - PCU_COMM_PACK(destRank, n); + PCUObj->Pack(destRank, n); //net for (vector::iterator pItr = nbItr->net.begin(); pItr != nbItr->net.end(); pItr++) { - PCU_COMM_PACK( destRank, *pItr); + PCUObj->Pack( destRank, *pItr); } } } @@ -84,124 +84,124 @@ namespace { return 0; } - void sendIntsToNeighbors(partInfo& part, vector& msg, int tag) { - PCU_Comm_Begin(); + void sendIntsToNeighbors(partInfo& part, vector& msg, int tag, pcu::PCU *PCUObj) { + PCUObj->Begin(); MIS_ITERATE(vector, part.adjPartIds, adjPartIdItr) { const int destRank = *adjPartIdItr; //pack msg tag - PCU_COMM_PACK(destRank, tag); + PCUObj->Pack(destRank, tag); //pack destination part Id - PCU_COMM_PACK(destRank, *adjPartIdItr); + PCUObj->Pack(destRank, *adjPartIdItr); //pack array length size_t n = msg.size(); - PCU_COMM_PACK(destRank, n); + PCUObj->Pack(destRank, n); //pack int array for ( vector::iterator pItr = msg.begin(); pItr != msg.end(); pItr++ ) { - PCU_COMM_PACK(destRank, *pItr); + PCUObj->Pack(destRank, *pItr); } } } - void unpackInts(vector& msg, int tag) { - const int rank = PCU_Comm_Self(); + void unpackInts(vector& msg, int tag, pcu::PCU *PCUObj) { + const int rank = PCUObj->Self(); //unpack msg tag int inTag; - PCU_COMM_UNPACK(inTag); + PCUObj->Unpack(inTag); MIS_FAIL_IF(tag != inTag, "tags do not match"); //unpack destination part Id int destPartId; - PCU_COMM_UNPACK(destPartId); + PCUObj->Unpack(destPartId); PCU_ALWAYS_ASSERT(rank == destPartId); //unpack array length size_t n; - PCU_COMM_UNPACK(n); + PCUObj->Unpack(n); //unpack int array int buff; for(size_t i=0; i < n; i++) { - PCU_COMM_UNPACK(buff); + PCUObj->Unpack(buff); msg.push_back(buff); } } - void recvIntsFromNeighbors(vector& msg, int tag) { - while(PCU_Comm_Listen()) - unpackInts(msg, tag); + void recvIntsFromNeighbors(vector& msg, int tag, pcu::PCU *PCUObj) { + while(PCUObj->Listen()) + unpackInts(msg, tag, PCUObj); } - int sendNetToNeighbors(partInfo& part) { - PCU_Comm_Begin(); + int sendNetToNeighbors(partInfo& part, pcu::PCU *PCUObj) { + PCUObj->Begin(); MIS_ITERATE(vector, part.adjPartIds, adjPartIdItr) { const int destRank = *adjPartIdItr; - PCU_COMM_PACK(destRank, part.id); //FIXME redundant - PCU_COMM_PACK(destRank, *adjPartIdItr); //FIXME redundant - PCU_COMM_PACK(destRank, part.randNum); + PCUObj->Pack(destRank, part.id); //FIXME redundant + PCUObj->Pack(destRank, *adjPartIdItr); //FIXME redundant + PCUObj->Pack(destRank, part.randNum); size_t netSz = part.net.size(); - PCU_COMM_PACK(destRank, netSz); + PCUObj->Pack(destRank, netSz); MIS_ITERATE(vector, part.net, pItr) - PCU_COMM_PACK(destRank, *pItr); + PCUObj->Pack(destRank, *pItr); } return 0; } - void unpackNet(vector& msg) { + void unpackNet(vector& msg, pcu::PCU *PCUObj) { int srcPartId; - PCU_COMM_UNPACK(srcPartId); - PCU_ALWAYS_ASSERT(PCU_Comm_Sender() == srcPartId); + PCUObj->Unpack(srcPartId); + PCU_ALWAYS_ASSERT(PCUObj->Sender() == srcPartId); int destPartId; - PCU_COMM_UNPACK(destPartId); - PCU_ALWAYS_ASSERT(PCU_Comm_Self() == destPartId); + PCUObj->Unpack(destPartId); + PCU_ALWAYS_ASSERT(PCUObj->Self() == destPartId); unsigned randNum; - PCU_COMM_UNPACK(randNum); + PCUObj->Unpack(randNum); adjPart ap; ap.partId = srcPartId; ap.randNum = randNum; size_t arrayLen; - PCU_COMM_UNPACK(arrayLen); + PCUObj->Unpack(arrayLen); PCU_ALWAYS_ASSERT(arrayLen > 0); int buff; for(size_t i=0; i < arrayLen; i++) { - PCU_COMM_UNPACK(buff); + PCUObj->Unpack(buff); ap.net.push_back(buff); } msg.push_back(ap); } - void recvNetsFromNeighbors(vector& msg) { - while( PCU_Comm_Listen() ) - unpackNet(msg); + void recvNetsFromNeighbors(vector& msg, pcu::PCU *PCUObj) { + while( PCUObj->Listen() ) + unpackNet(msg, PCUObj); } - void unpackAdjPart(vector& msg) { - const int rank = PCU_Comm_Self(); + void unpackAdjPart(vector& msg, pcu::PCU *PCUObj) { + const int rank = PCUObj->Self(); //unpack destination part Id int destPartId; - PCU_COMM_UNPACK(destPartId); + PCUObj->Unpack(destPartId); PCU_ALWAYS_ASSERT(rank == destPartId); //unpack adjacent part Id int adjPartId; - PCU_COMM_UNPACK(adjPartId); + PCUObj->Unpack(adjPartId); //unpack random number unsigned randNum; - PCU_COMM_UNPACK(randNum); + PCUObj->Unpack(randNum); adjPart ap; ap.partId = adjPartId; @@ -209,21 +209,21 @@ namespace { //unpack net size size_t n; - PCU_COMM_UNPACK(n); + PCUObj->Unpack(n); //unpack net int pid; for(size_t i=0; i < n; i++) { - PCU_COMM_UNPACK(pid); + PCUObj->Unpack(pid); ap.net.push_back(pid); } msg.push_back(ap); } - void recvAdjNetsFromNeighbors(vector& msg) { - while( PCU_Comm_Receive() ) - unpackAdjPart(msg); + void recvAdjNetsFromNeighbors(vector& msg, pcu::PCU *PCUObj) { + while( PCUObj->Receive() ) + unpackAdjPart(msg, PCUObj); } int minRandNum(mapiuItr first, mapiuItr last) { @@ -302,25 +302,25 @@ namespace { * @param part (InOut) local part * @return 0 on success, non-zero otherwise */ - int constructNetGraph(partInfo& part) { + int constructNetGraph(partInfo& part, pcu::PCU *PCUObj) { stable_sort(part.adjPartIds.begin(), part.adjPartIds.end()); setNodeStateInGraph(part); // send net to each neighbor - int ierr = sendNetToNeighbors(part); + int ierr = sendNetToNeighbors(part, PCUObj); if (ierr != 0) return ierr; - PCU_Comm_Send(); + PCUObj->Send(); vector nbNet; - recvNetsFromNeighbors(nbNet); + recvNetsFromNeighbors(nbNet, PCUObj); // get the random num associated with adjacent nets part.addNetNeighbors(nbNet); - ierr = sendAdjNetsToNeighbors(part, nbNet); + ierr = sendAdjNetsToNeighbors(part, nbNet, PCUObj); if (ierr != 0) return ierr; - PCU_Comm_Send(); + PCUObj->Send(); vector nbAdjNets; - recvAdjNetsFromNeighbors(nbAdjNets); + recvAdjNetsFromNeighbors(nbAdjNets, PCUObj); // get the random num associated with adjacent nets part.addNetNeighbors(nbAdjNets); @@ -388,13 +388,13 @@ void misFinalize() {} * Setting isNeighbors true supports computing on the partition model graph * as opposed to the netgraph. */ -int mis(partInfo& part, bool randNumsPredefined,bool isNeighbors) { - PCU_ALWAYS_ASSERT(PCU_Comm_Initialized()); +int mis(partInfo& part, pcu::PCU *PCUObj, bool randNumsPredefined, bool isNeighbors) { + PCU_ALWAYS_ASSERT(PCUObj != nullptr); if (false == randNumsPredefined) - setRandomNum(part); + setRandomNum(part, PCUObj); - int ierr = constructNetGraph(part); + int ierr = constructNetGraph(part, PCUObj); if (ierr != 0) return ierr; vector nodesRemoved; @@ -429,9 +429,9 @@ int mis(partInfo& part, bool randNumsPredefined,bool isNeighbors) { //--------------ROUND tag++; - sendIntsToNeighbors(part, nodesToRemove, tag); - PCU_Comm_Send(); - recvIntsFromNeighbors(rmtNodesToRemove, tag); + sendIntsToNeighbors(part, nodesToRemove, tag, PCUObj); + PCUObj->Send(); + recvIntsFromNeighbors(rmtNodesToRemove, tag, PCUObj); if (true == part.isInNetGraph && @@ -451,15 +451,15 @@ int mis(partInfo& part, bool randNumsPredefined,bool isNeighbors) { //--------------ROUND tag++; - sendIntsToNeighbors(part, nodesRemoved, tag); - PCU_Comm_Send(); - recvIntsFromNeighbors(rmtNodesToRemove, tag); + sendIntsToNeighbors(part, nodesRemoved, tag, PCUObj); + PCUObj->Send(); + recvIntsFromNeighbors(rmtNodesToRemove, tag, PCUObj); removeNodes(part, rmtNodesToRemove); nodesRemoved.clear(); rmtNodesToRemove.clear(); - numNodesAdded = PCU_Add_Int(numNodesAdded); + numNodesAdded = PCUObj->Add(numNodesAdded); } while (numNodesAdded > 0); return isInMis; diff --git a/parma/diffMC/maximalIndependentSet/test/testMIS.cc b/parma/diffMC/maximalIndependentSet/test/testMIS.cc index f4b7bb481..1c1c5f4a6 100644 --- a/parma/diffMC/maximalIndependentSet/test/testMIS.cc +++ b/parma/diffMC/maximalIndependentSet/test/testMIS.cc @@ -6,7 +6,6 @@ #include "math.h" #include #include -#include "PCU.h" #include "parma_commons.h" using std::set; @@ -273,9 +272,9 @@ are removed: |MIS| = 2 */ -int test_2dStencil(const int rank, const int totNumParts, +int test_2dStencil(const int rank, const int totNumParts, pcu::PCU *PCUObj bool randNumsPredefined = false, bool isNeighbors = false) { - if( !PCU_Comm_Self() ) + if( !PCUObj->Self() ) status("test_2dStencil - totNumParts: %d\n", totNumParts); const int sqrtTotNumParts = floor(sqrt(totNumParts)); @@ -299,16 +298,16 @@ int test_2dStencil(const int rank, const int totNumParts, //sanity check checkAdjPartsandNets(part, totNumParts); - const double t1 = PCU_Time(); - int isInMis = mis(part, randNumsPredefined,isNeighbors); - double elapsedTime = PCU_Time() - t1; - PCU_Max_Doubles(&elapsedTime, 1); + const double t1 = pcu::Time(); + int isInMis = mis(part, PCUObj, randNumsPredefined,isNeighbors); + double elapsedTime = pcu::Time() - t1; + PCUObj->Max(&elapsedTime, 1); - if( !PCU_Comm_Self() ) + if( !PCUObj->Self() ) status("elapsed time (seconds) = %f \n", elapsedTime); int* globalIsInMIS = NULL; - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) globalIsInMIS = new int[totNumParts]; MPI_Gather(&isInMis, 1, MPI_INT, globalIsInMIS, 1, MPI_INT, 0, MPI_COMM_WORLD); @@ -363,10 +362,10 @@ int test_2dStencil(const int rank, const int totNumParts, * @param randNumsPredefined (In) T:rand nums predefined, F:rand nums not predefined * @return 0 on success, non-zero otherwise */ -int test_StarA(const int rank, const int totNumParts, +int test_StarA(const int rank, const int totNumParts, pcu::PCU *PCUObj, bool randNumsPredefined = false) { char dbgMsg[256]; - if( !PCU_Comm_Self() ) + if( !PCUObj->Self() ) status("test_StarA - totNumParts: %d\n", totNumParts); partInfo part; @@ -414,7 +413,7 @@ int test_StarA(const int rank, const int totNumParts, Print ::iterator > (cout, dbgMsg, part.net.begin(), part.net.end(), std::string(", ")); - int isInMIS = mis(part, randNumsPredefined); + int isInMIS = mis(part, PCUObj, randNumsPredefined); int* globalIsInMIS = NULL; if (rank == 0) { @@ -466,13 +465,13 @@ randNums 5 1 13 6 */ int test_4partsA(const int rank, const int totNumParts, - bool predefinedRandNums) { + bool predefinedRandNums, pcu::PCU *PCUObj) { if (4 != totNumParts) { MIS_FAIL("totNumParts must be 4\n"); return 1; } - if( !PCU_Comm_Self() ) + if( !PCUObj->Self() ) status("test_4partsA - totNumParts: %d\n", totNumParts); partInfo part; @@ -511,7 +510,7 @@ int test_4partsA(const int rank, const int totNumParts, vector parts; parts.push_back(part); - int isInMis = mis(parts[0], predefinedRandNums); + int isInMis = mis(parts[0], PCUObj, predefinedRandNums); int* globalIsInMIS = NULL; if (!rank) @@ -542,13 +541,13 @@ int test_4partsA(const int rank, const int totNumParts, } int test_4partsB(const int rank, const int totNumParts, - bool predefinedRandNums) { + bool predefinedRandNums, pcu::PCU *PCUObj) { if (4 != totNumParts) { MIS_FAIL("totNumParts must be 4\n"); return 1; } - if( !PCU_Comm_Self() ) + if( !PCUObj->Self() ) status("test_4partsAndNums - totNumParts: %d\n", totNumParts); partInfo part; @@ -583,7 +582,7 @@ int test_4partsB(const int rank, const int totNumParts, vector parts; parts.push_back(part); - int isInMis = mis(parts[0], predefinedRandNums); + int isInMis = mis(parts[0], PCUObj, predefinedRandNums); int* globalIsInMIS = NULL; if (rank == 0) { @@ -615,13 +614,13 @@ int test_4partsB(const int rank, const int totNumParts, } int test_4partsC(const int rank, const int totNumParts, - bool predefinedRandNums) { + bool predefinedRandNums, pcu::PCU *PCUObj) { if (4 != totNumParts) { MIS_FAIL("totNumParts must be 4\n"); return 1; } - if( !PCU_Comm_Self() ) + if( !PCUObj->Self() ) status("test_4partsA - totNumParts: %d\n", totNumParts); partInfo part; @@ -651,7 +650,7 @@ int test_4partsC(const int rank, const int totNumParts, vector parts; parts.push_back(part); - int isInMis = mis(parts[0], predefinedRandNums); + int isInMis = mis(parts[0], PCUObj, predefinedRandNums); int* globalIsInMIS = NULL; if (rank == 0) { @@ -711,12 +710,11 @@ bool getBool(int& isDebug) { int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); - PCU_Protect(); - int rank; - PCU_Comm_Rank(&rank); - int commSize; - PCU_Comm_Size(&commSize); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::Protect(); + int rank = PCUObj.Self(); + int commSize = PCUObj.Peers(); int opt; int debugMode = 0; @@ -748,7 +746,7 @@ int main(int argc, char** argv) { testNum = broadcastInt(testNum); - randNumSeed = broadcastInt(randNumSeed) + PCU_Comm_Self(); + randNumSeed = broadcastInt(randNumSeed) + PCUObj.Self(); bool predefinedRandNums = getBool(iPredefinedRandNums); mis_init(randNumSeed, getBool(debugMode)); @@ -765,22 +763,22 @@ int main(int argc, char** argv) { return 0; break; case 0: - ierr = test_4partsA(rank, commSize, predefinedRandNums); + ierr = test_4partsA(rank, commSize, predefinedRandNums, &PCUObj); break; case 1: - ierr = test_4partsB(rank, commSize, predefinedRandNums); + ierr = test_4partsB(rank, commSize, predefinedRandNums, &PCUObj); break; case 2: - ierr = test_4partsC(rank, commSize, predefinedRandNums); + ierr = test_4partsC(rank, commSize, predefinedRandNums, &PCUObj); break; case 3: - ierr = test_StarA(rank, commSize); + ierr = test_StarA(rank, commSize, &PCUObj); break; case 4: - ierr = test_2dStencil(rank, commSize); + ierr = test_2dStencil(rank, commSize, &PCUObj); break; case 5: - ierr = test_2dStencil(rank,commSize,false,true); + ierr = test_2dStencil(rank,commSize,&PCUObj,false,true); break; } @@ -792,7 +790,7 @@ int main(int argc, char** argv) { } } misFinalize(); - PCU_Comm_Free(); + } MPI_Finalize(); return 0; } diff --git a/parma/diffMC/parma_balancer.cc b/parma/diffMC/parma_balancer.cc index 742575b19..14e3a735f 100644 --- a/parma/diffMC/parma_balancer.cc +++ b/parma/diffMC/parma_balancer.cc @@ -1,12 +1,11 @@ -#include #include "parma_balancer.h" #include "parma_monitor.h" #include "parma_graphDist.h" #include "parma_commons.h" namespace { - void printTiming(const char* type, int steps, double tol, double time) { - if (!PCU_Comm_Self()) + void printTiming(const char* type, int steps, double tol, double time, pcu::PCU *PCUObj) { + if (!PCUObj->Self()) parmaCommons::status("%s balanced in %d steps to %f in %f seconds\n", type, steps, tol, time); } @@ -33,11 +32,11 @@ namespace parma { } } void Balancer::balance(apf::MeshTag* wtag, double tolerance) { - if( 1 == PCU_Comm_Peers() ) return; + if( 1 == mesh->getPCU()->Peers() ) return; int step = 0; - double t0 = PCU_Time(); + double t0 = pcu::Time(); while (runStep(wtag,tolerance) && step++ < maxStep); - printTiming(name, step, tolerance, PCU_Time()-t0); + printTiming(name, step, tolerance, pcu::Time()-t0, mesh->getPCU()); } void Balancer::monitorUpdate(double v, Slope* s, Average* a) { s->push(v); diff --git a/parma/diffMC/parma_centroidDiffuser.cc b/parma/diffMC/parma_centroidDiffuser.cc index cdfa82c6a..a15a294d9 100644 --- a/parma/diffMC/parma_centroidDiffuser.cc +++ b/parma/diffMC/parma_centroidDiffuser.cc @@ -1,4 +1,3 @@ -#include #include #include "parma.h" #include "parma_step.h" diff --git a/parma/diffMC/parma_centroidSelector.cc b/parma/diffMC/parma_centroidSelector.cc index d3ffd5711..63e37f07d 100644 --- a/parma/diffMC/parma_centroidSelector.cc +++ b/parma/diffMC/parma_centroidSelector.cc @@ -1,4 +1,3 @@ -#include "PCU.h" #include #include "parma_selector.h" #include "parma_targets.h" diff --git a/parma/diffMC/parma_centroids.cc b/parma/diffMC/parma_centroids.cc index 00aee0ec6..aee591337 100644 --- a/parma/diffMC/parma_centroids.cc +++ b/parma/diffMC/parma_centroids.cc @@ -1,5 +1,4 @@ #include -#include #include "parma_centroids.h" #include "parma_sides.h" @@ -43,18 +42,18 @@ namespace parma { return x / weight; } - void Centroids::init(apf::Mesh*, Sides* s) { - PCU_Comm_Begin(); + void Centroids::init(apf::Mesh* m, Sides* s) { + m->getPCU()->Begin(); const Sides::Item* side; s->begin(); while( (side = s->iterate()) ) - PCU_COMM_PACK(side->first, centroid); + m->getPCU()->Pack(side->first, centroid); s->end(); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { apf::Vector3 otherCentroid; - PCU_COMM_UNPACK(otherCentroid); - set(PCU_Comm_Sender(), otherCentroid); + m->getPCU()->Unpack(otherCentroid); + set(m->getPCU()->Sender(), otherCentroid); } } } //end namespace diff --git a/parma/diffMC/parma_commons.cc b/parma/diffMC/parma_commons.cc index afaa72812..80b228ab7 100644 --- a/parma/diffMC/parma_commons.cc +++ b/parma/diffMC/parma_commons.cc @@ -1,5 +1,4 @@ #include "parma_commons.h" -#include "PCU.h" #include #include #include @@ -45,9 +44,9 @@ int parmaCommons::isMore(double a, double b) { return 0; } -void parmaCommons::printElapsedTime(const char* fn, double elapsed) { - elapsed = PCU_Max_Double(elapsed); - if( !PCU_Comm_Self() ) +void parmaCommons::printElapsedTime(const char* fn, double elapsed, pcu::PCU *PCUObj) { + elapsed = PCUObj->Max(elapsed); + if( !PCUObj->Self() ) status("%s elapsed time %lf seconds\n", fn, elapsed); } diff --git a/parma/diffMC/parma_commons.h b/parma/diffMC/parma_commons.h index 48c2ce755..ab2fc588d 100644 --- a/parma/diffMC/parma_commons.h +++ b/parma/diffMC/parma_commons.h @@ -2,6 +2,7 @@ #define PARMA_COMMONS_H_ #include "apfArray.h" +#include "PCU.h" namespace parmaCommons { @@ -16,7 +17,7 @@ template int isEqlArr(type1 a, type2 b) { int isLess(double a, double b); int isMore(double a, double b); -void printElapsedTime(const char* fn, double elapsed); +void printElapsedTime(const char* fn, double elapsed, pcu::PCU *PCUObj); void debug(bool isActive, const char* fmt,...); void status(const char* fmt,...); diff --git a/parma/diffMC/parma_components.cc b/parma/diffMC/parma_components.cc index 59725f4b2..37497a537 100644 --- a/parma/diffMC/parma_components.cc +++ b/parma/diffMC/parma_components.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include "parma_components.h" #include "parma_meshaux.h" #include "parma_convert.h" @@ -148,7 +147,7 @@ namespace parma { for(unsigned i=0; igetPCU()->DebugPrint("core %u is empty... assigning core to bdry\n", i); core[i] = bdry[i]; } } diff --git a/parma/diffMC/parma_dcpart.cc b/parma/diffMC/parma_dcpart.cc index b7d3d1278..e43844582 100644 --- a/parma/diffMC/parma_dcpart.cc +++ b/parma/diffMC/parma_dcpart.cc @@ -1,4 +1,3 @@ -#include "PCU.h" #include "parma_dcpart.h" #include "parma_commons.h" #include "parma_meshaux.h" @@ -100,7 +99,7 @@ void dcPart::reset() { } unsigned dcPart::numDisconnectedComps() { - double t1 = PCU_Time(); + double t1 = pcu::Time(); reset(); unsigned numDc = 0; size_t count = 0; @@ -109,7 +108,7 @@ unsigned dcPart::numDisconnectedComps() { while( count != numElms ) { unsigned sz = walkPart(numDc); unsigned nbor = maxContactNeighbor(numDc); - if( nbor != self || PCU_Comm_Peers() == 1 ) { + if( nbor != self || m->getPCU()->Peers() == 1 ) { dcCompSz.push_back(sz); dcCompNbor.push_back(nbor); numDc++; @@ -120,7 +119,7 @@ unsigned dcPart::numDisconnectedComps() { count += sz; } if( verbose ) - parmaCommons::printElapsedTime(__func__, PCU_Time() - t1); + parmaCommons::printElapsedTime(__func__, pcu::Time() - t1, m->getPCU()); PCU_ALWAYS_ASSERT(numDc+numIso >= 1); return (numDc+numIso)-1; } diff --git a/parma/diffMC/parma_dcpartFixer.cc b/parma/diffMC/parma_dcpartFixer.cc index a778c2ff3..9576e7eed 100644 --- a/parma/diffMC/parma_dcpartFixer.cc +++ b/parma/diffMC/parma_dcpartFixer.cc @@ -1,4 +1,3 @@ -#include "PCU.h" #include "parma_dcpart.h" #include "parma_commons.h" #include "parma_convert.h" @@ -8,11 +7,11 @@ typedef std::map muu; namespace { - bool isInMis(muu& mt) { - unsigned seed = TO_UINT(PCU_Comm_Self()+1); + bool isInMis(muu& mt, pcu::PCU *PCUObj) { + unsigned seed = TO_UINT(PCUObj->Self()+1); mis_init(seed); misLuby::partInfo part; - part.id = PCU_Comm_Self(); + part.id = PCUObj->Self(); std::set targets; APF_ITERATE(muu, mt, mtItr) { int peer = TO_INT(mtItr->second); @@ -23,7 +22,7 @@ namespace { } } part.net.push_back(part.id); - return mis(part,false,true); + return mis(part,PCUObj,false,true); } } @@ -41,7 +40,7 @@ class dcPartFixer::PartFixer : public dcPart { int totNumDc() { int ndc = TO_INT(numDisconnectedComps()); - return PCU_Add_Int(ndc); + return m->getPCU()->Add(ndc); } void setupPlan(muu& dcCompTgts, apf::Migration* plan) { @@ -64,11 +63,11 @@ class dcPartFixer::PartFixer : public dcPart { * are tagged */ void fix() { - double t1 = PCU_Time(); + double t1 = pcu::Time(); int loop = 0; int ndc = 0; while( (ndc = totNumDc()) && loop++ < 50 ) { - double t2 = PCU_Time(); + double t2 = pcu::Time(); muu dcCompTgts; unsigned maxSz = 0; @@ -81,18 +80,18 @@ class dcPartFixer::PartFixer : public dcPart { dcCompTgts[i] = getCompPeer(i); PCU_ALWAYS_ASSERT( dcCompTgts.size() == getNumComps()-1 ); apf::Migration* plan = new apf::Migration(m); - if ( isInMis(dcCompTgts) ) + if ( isInMis(dcCompTgts, m->getPCU()) ) setupPlan(dcCompTgts, plan); reset(); - double t3 = PCU_Time(); + double t3 = pcu::Time(); m->migrate(plan); - if( ! PCU_Comm_Self() && vb) + if( ! m->getPCU()->Self() && vb) parmaCommons::status( "loop %d components %d seconds %.3f %.3f\n", - loop, ndc, t3-t2, PCU_Time()-t3); + loop, ndc, t3-t2, pcu::Time()-t3); } - parmaCommons::printElapsedTime(__func__, PCU_Time() - t1); + parmaCommons::printElapsedTime(__func__, pcu::Time() - t1, m->getPCU()); } }; diff --git a/parma/diffMC/parma_dijkstra.cc b/parma/diffMC/parma_dijkstra.cc index 22e27c118..30d2b2966 100644 --- a/parma/diffMC/parma_dijkstra.cc +++ b/parma/diffMC/parma_dijkstra.cc @@ -1,6 +1,5 @@ #include #include -#include #include #include "parma_dijkstra.h" #include "parma_meshaux.h" diff --git a/parma/diffMC/parma_edgeEqVtxSelector.cc b/parma/diffMC/parma_edgeEqVtxSelector.cc index 048860b2b..1d517774b 100644 --- a/parma/diffMC/parma_edgeEqVtxSelector.cc +++ b/parma/diffMC/parma_edgeEqVtxSelector.cc @@ -1,6 +1,5 @@ #include #include -#include #include "parma_vtxSelector.h" #include "parma_targets.h" #include "parma_weights.h" @@ -138,16 +137,16 @@ namespace { typedef std::set MigrComm; - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); APF_ITERATE(parma::Mid, sendingVtx, s) - PCU_COMM_PACK(s->first, s->second); - PCU_Comm_Send(); + mesh->getPCU()->Pack(s->first, s->second); + mesh->getPCU()->Send(); MigrComm incoming; //map double w; - while (PCU_Comm_Listen()) { - PCU_COMM_UNPACK(w); - incoming.insert(Migr(PCU_Comm_Sender(),w)); + while (mesh->getPCU()->Listen()) { + mesh->getPCU()->Unpack(w); + incoming.insert(Migr(mesh->getPCU()->Sender(),w)); } double selfW = parma::getWeight(mesh,wtag,0); @@ -165,15 +164,15 @@ namespace { totW += accept[in->first]; } - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); APF_ITERATE(parma::Mid, accept, a) - PCU_COMM_PACK(a->first, a->second); - PCU_Comm_Send(); + mesh->getPCU()->Pack(a->first, a->second); + mesh->getPCU()->Send(); parma::Mid* capacity = new parma::Mid; double outw; - while (PCU_Comm_Listen()) { - PCU_COMM_UNPACK(outw); - (*capacity)[PCU_Comm_Sender()] = outw; + while (mesh->getPCU()->Listen()) { + mesh->getPCU()->Unpack(outw); + (*capacity)[mesh->getPCU()->Sender()] = outw; } return capacity; } diff --git a/parma/diffMC/parma_elmBalancer.cc b/parma/diffMC/parma_elmBalancer.cc index 757b2d25b..e897a7780 100644 --- a/parma/diffMC/parma_elmBalancer.cc +++ b/parma/diffMC/parma_elmBalancer.cc @@ -1,4 +1,3 @@ -#include #include #include "parma.h" #include "parma_step.h" @@ -18,14 +17,14 @@ namespace { ElmBalancer(apf::Mesh* m, double f, int v) : Balancer(m, f, v, "elements") { parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = parma::avgSharedSides(s); + sideTol = parma::avgSharedSides(s, mesh->getPCU()); delete s; } bool runStep(apf::MeshTag* wtag, double tolerance) { const double maxElmImb = Parma_GetWeightedEntImbalance(mesh, wtag, mesh->getDimension()); parma::Sides* s = parma::makeVtxSides(mesh); - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); parma::Weights* w = parma::makeEntWeights(mesh, wtag, s, mesh->getDimension()); parma::Targets* t = parma::makeTargets(s, w, factor); @@ -33,7 +32,7 @@ namespace { monitorUpdate(maxElmImb, iS, iA); monitorUpdate(avgSides, sS, sA); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("elmImb %f avgSides %f\n", maxElmImb, avgSides); parma::BalOrStall* stopper = new parma::BalOrStall(iA, sA, sideTol*.001, verbose); @@ -46,7 +45,7 @@ namespace { apf::Balancer* Parma_MakeElmBalancer(apf::Mesh* m, double stepFactor, int verbosity) { - if( !PCU_Comm_Self() && verbosity ) + if( !m->getPCU()->Self() && verbosity ) status("stepFactor %.3f\n", stepFactor); return new ElmBalancer(m, stepFactor, verbosity); } diff --git a/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc b/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc index 75e9eb81f..1d05b0eab 100644 --- a/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc +++ b/parma/diffMC/parma_elmLtVtxEdgeBalancer.cc @@ -1,4 +1,3 @@ -#include #include #include #include "parma_balancer.h" @@ -24,15 +23,15 @@ namespace { : Balancer(m, f, v, "elements") { maxVtx = maxV; maxEdge = maxE; - if( !PCU_Comm_Self() && verbose ) { + if( !mesh->getPCU()->Self() && verbose ) { status("stepFactor %.3f\n", f); status("maxVtx %.3f\n", maxVtx); status("maxEdge %.3f\n", maxEdge); } parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = TO_INT(parma::avgSharedSides(s)); + sideTol = TO_INT(parma::avgSharedSides(s, mesh->getPCU())); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { @@ -49,11 +48,10 @@ namespace { delete w[1]; parma::Selector* sel = parma::makeElmLtVtxEdgeSelector(mesh, wtag, maxVtx, maxEdge); - - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); monitorUpdate(maxElmImb, iS, iA); monitorUpdate(avgSides, sS, sA); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("elmImb %f avgSides %f\n", maxElmImb, avgSides); parma::BalOrStall* stopper = new parma::BalOrStall(iA, sA, sideTol*.001, verbose); diff --git a/parma/diffMC/parma_elmLtVtxEdgeSelector.cc b/parma/diffMC/parma_elmLtVtxEdgeSelector.cc index 9fddb6d81..681d43880 100644 --- a/parma/diffMC/parma_elmLtVtxEdgeSelector.cc +++ b/parma/diffMC/parma_elmLtVtxEdgeSelector.cc @@ -1,6 +1,5 @@ #include #include -#include #include "parma_vtxSelector.h" #include "parma_targets.h" #include "parma_weights.h" @@ -137,21 +136,21 @@ namespace { } //send vtx and edge weight - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); APF_ITERATE(PeerEntSet, peerVerts, pv) { const int dest = pv->first; double vw = weight(peerVerts[dest]); - PCU_COMM_PACK(dest, vw); + mesh->getPCU()->Pack(dest, vw); double ew = weight(peerEdges[dest]); - PCU_COMM_PACK(dest, ew); + mesh->getPCU()->Pack(dest, ew); } - PCU_Comm_Send(); + mesh->getPCU()->Send(); MigrComm incoming; double vw, ew; - while (PCU_Comm_Listen()) { - PCU_COMM_UNPACK(vw); - PCU_COMM_UNPACK(ew); - incoming.insert(Migr(PCU_Comm_Sender(),vw,ew)); //Migr ctor does not exist + while (mesh->getPCU()->Listen()) { + mesh->getPCU()->Unpack(vw); + mesh->getPCU()->Unpack(ew); + incoming.insert(Migr(mesh->getPCU()->Sender(),vw,ew)); //Migr ctor does not exist } Midd accept; @@ -182,18 +181,18 @@ namespace { totW[1] += accept[nbor].b; } - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); APF_ITERATE(Midd, accept, acc) { - PCU_COMM_PACK(acc->first, acc->second.a); - PCU_COMM_PACK(acc->first, acc->second.b); + mesh->getPCU()->Pack(acc->first, acc->second.a); + mesh->getPCU()->Pack(acc->first, acc->second.b); } - PCU_Comm_Send(); + mesh->getPCU()->Send(); Midd* capacity = new Midd; dtwo outw; - while (PCU_Comm_Listen()) { - PCU_COMM_UNPACK(outw.a); - PCU_COMM_UNPACK(outw.b); - (*capacity)[PCU_Comm_Sender()] = outw; + while (mesh->getPCU()->Listen()) { + mesh->getPCU()->Unpack(outw.a); + mesh->getPCU()->Unpack(outw.b); + (*capacity)[mesh->getPCU()->Sender()] = outw; } return capacity; } diff --git a/parma/diffMC/parma_elmLtVtxEdgeTargets.cc b/parma/diffMC/parma_elmLtVtxEdgeTargets.cc index 64ca0a2b6..067d9a81b 100644 --- a/parma/diffMC/parma_elmLtVtxEdgeTargets.cc +++ b/parma/diffMC/parma_elmLtVtxEdgeTargets.cc @@ -1,4 +1,3 @@ -#include #include "parma_sides.h" #include "parma_weights.h" #include "parma_targets.h" diff --git a/parma/diffMC/parma_entWeights.cc b/parma/diffMC/parma_entWeights.cc index 66c9dc2d7..ca085593d 100644 --- a/parma/diffMC/parma_entWeights.cc +++ b/parma/diffMC/parma_entWeights.cc @@ -1,17 +1,16 @@ #include -#include #include "parma_entWeights.h" #include "parma_sides.h" namespace parma { double getMaxWeight(apf::Mesh* m, apf::MeshTag* w, int entDim) { double locW = getWeight(m,w,entDim); - return PCU_Max_Double(locW); + return m->getPCU()->Max(locW); } double getAvgWeight(apf::Mesh* m, apf::MeshTag* w, int entDim) { double locW = getWeight(m,w,entDim); - return PCU_Add_Double(locW) / PCU_Comm_Peers(); + return m->getPCU()->Add(locW) / m->getPCU()->Peers(); } double getWeight(apf::Mesh* m, apf::MeshTag* w, int entDim) { @@ -25,17 +24,17 @@ namespace parma { return sum; } - void getImbalance(Weights* w, double& imb, double& avg) { + void getImbalance(Weights* w, double& imb, double& avg, pcu::PCU *PCUObj) { double sum, max; sum = max = w->self(); - sum = PCU_Add_Double(sum); - max = PCU_Max_Double(max); - avg = sum/PCU_Comm_Peers(); + sum = PCUObj->Add(sum); + max = PCUObj->Max(max); + avg = sum/PCUObj->Peers(); imb = max/avg; } - double getMaxWeight(Weights* w) { - return PCU_Max_Double(w->self()); + double getMaxWeight(Weights* w, pcu::PCU *PCUObj) { + return PCUObj->Max(w->self()); } double getEntWeight(apf::Mesh* m, apf::MeshEntity* e, apf::MeshTag* w) { @@ -63,18 +62,18 @@ namespace parma { return entW; } - void EntWeights::init(apf::Mesh*, apf::MeshTag*, Sides* s) { - PCU_Comm_Begin(); + void EntWeights::init(apf::Mesh* m, apf::MeshTag*, Sides* s) { + m->getPCU()->Begin(); const Sides::Item* side; s->begin(); while( (side = s->iterate()) ) - PCU_COMM_PACK(side->first, weight); + m->getPCU()->Pack(side->first, weight); s->end(); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { double otherWeight; - PCU_COMM_UNPACK(otherWeight); - set(PCU_Comm_Sender(), otherWeight); + m->getPCU()->Unpack(otherWeight); + set(m->getPCU()->Sender(), otherWeight); } } Weights* makeEntWeights(apf::Mesh* m, apf::MeshTag* w, Sides* s, int dim) { diff --git a/parma/diffMC/parma_ghost.cc b/parma/diffMC/parma_ghost.cc index da42857e0..edd1fe68f 100644 --- a/parma/diffMC/parma_ghost.cc +++ b/parma/diffMC/parma_ghost.cc @@ -1,5 +1,4 @@ #include -#include #include "parma.h" #include "parma_balancer.h" #include "parma_step.h" @@ -21,15 +20,15 @@ namespace { : Balancer(m, f, v, "ghostElms"), layers(l) { parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = TO_INT(parma::avgSharedSides(s)); + sideTol = TO_INT(parma::avgSharedSides(s, mesh->getPCU())); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { parma::Sides* s = parma::makeVtxSides(mesh); - double avgSides = parma::avgSharedSides(s); - if( !PCU_Comm_Self() && verbose ) + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); + if( !mesh->getPCU()->Self() && verbose ) status("avgSides %f\n", avgSides); parma::GhostWeights* gw = @@ -39,13 +38,13 @@ namespace { destroyGhostWeights(gw); double vtxImb, vtxAvg; - parma::getImbalance(vtxW, vtxImb, vtxAvg); - if( !PCU_Comm_Self() && verbose ) + parma::getImbalance(vtxW, vtxImb, vtxAvg, mesh->getPCU()); + if( !mesh->getPCU()->Self() && verbose ) status("vtx imbalance %.3f avg %.3f\n", vtxImb, vtxAvg); delete vtxW; double elmImb, elmAvg; - parma::getImbalance(elmW, elmImb, elmAvg); + parma::getImbalance(elmW, elmImb, elmAvg, mesh->getPCU()); monitorUpdate(elmImb, iS, iA); monitorUpdate(avgSides, sS, sA); @@ -71,15 +70,15 @@ namespace { : Balancer(m, f, v, "ghostVtxLtElms"), layers(l), maxElmW(0), stepNum(0) { parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = TO_INT(parma::avgSharedSides(s)); + sideTol = TO_INT(parma::avgSharedSides(s, mesh->getPCU())); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { parma::Sides* s = parma::makeVtxSides(mesh); - double avgSides = parma::avgSharedSides(s); - if( !PCU_Comm_Self() && verbose ) + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); + if( !mesh->getPCU()->Self() && verbose ) status("avgSides %f\n", avgSides); parma::GhostWeights* gw = @@ -90,15 +89,15 @@ namespace { destroyGhostWeights(gw); double elmImb, elmAvg; - parma::getImbalance(elmW,elmImb,elmAvg); + parma::getImbalance(elmW,elmImb,elmAvg,mesh->getPCU()); double edgeImb, edgeAvg; - parma::getImbalance(edgeW, edgeImb, edgeAvg); - if( !PCU_Comm_Self() && verbose ) { + parma::getImbalance(edgeW, edgeImb, edgeAvg, mesh->getPCU()); + if( !mesh->getPCU()->Self() && verbose ) { status("elm imbalance %.3f avg %.3f\n", elmImb, elmAvg); status("edge imbalance %.3f avg %.3f\n", edgeImb, edgeAvg); } if( !stepNum ) //FIXME need to set the imbalance at the beginning for the primary entity - maxElmW = parma::getMaxWeight(elmW); + maxElmW = parma::getMaxWeight(elmW, mesh->getPCU()); delete edgeW; monitorUpdate(elmImb, iS, iA); diff --git a/parma/diffMC/parma_ghostElement.cc b/parma/diffMC/parma_ghostElement.cc index 6d11a6cc5..1514d413b 100644 --- a/parma/diffMC/parma_ghostElement.cc +++ b/parma/diffMC/parma_ghostElement.cc @@ -1,5 +1,4 @@ #include -#include #include "parma.h" #include "parma_balancer.h" #include "parma_step.h" @@ -21,15 +20,15 @@ namespace { : Balancer(m, f, v, "ghostEdges") { parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = TO_INT(parma::avgSharedSides(s)); + sideTol = TO_INT(parma::avgSharedSides(s, mesh->getPCU())); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { parma::Sides* s = parma::makeVtxSides(mesh); - double avgSides = parma::avgSharedSides(s); - if( !PCU_Comm_Self() && verbose ) + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); + if( !mesh->getPCU()->Self() && verbose ) status("avgSides %f\n", avgSides); parma::GhostWeights* gw = @@ -40,9 +39,9 @@ namespace { destroyGhostWeights(gw); double faceImb, faceAvg, elmImb, elmAvg; - parma::getImbalance(faceW, faceImb, faceAvg); - parma::getImbalance(elmW, elmImb, elmAvg); - if( !PCU_Comm_Self() && verbose ) { + parma::getImbalance(faceW, faceImb, faceAvg, mesh->getPCU()); + parma::getImbalance(elmW, elmImb, elmAvg, mesh->getPCU()); + if( !mesh->getPCU()->Self() && verbose ) { status("face imbalance %.3f avg %.3f\n", faceImb, faceAvg); status("elm imbalance %.3f avg %.3f\n", elmImb, elmAvg); } @@ -50,7 +49,7 @@ namespace { delete elmW; double edgeImb, edgeAvg; - parma::getImbalance(edgeW, edgeImb, edgeAvg); + parma::getImbalance(edgeW, edgeImb, edgeAvg, mesh->getPCU()); monitorUpdate(edgeImb, iS, iA); monitorUpdate(avgSides, sS, sA); diff --git a/parma/diffMC/parma_ghostMPAS.cc b/parma/diffMC/parma_ghostMPAS.cc index 5c2d6c806..ab4a6afff 100644 --- a/parma/diffMC/parma_ghostMPAS.cc +++ b/parma/diffMC/parma_ghostMPAS.cc @@ -1,5 +1,4 @@ #include -#include #include #include "parma.h" #include "parma_balancer.h" @@ -18,9 +17,9 @@ namespace { : Balancer(m, f, v, "ghosts"), layers(l), bridge(b) { parma::Sides* s = parma::makeElmBdrySides(mesh); - sideTol = static_cast(parma::avgSharedSides(s)); + sideTol = static_cast(parma::avgSharedSides(s, mesh->getPCU())); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) lion_oprint(1, "sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { @@ -28,10 +27,10 @@ namespace { const double maxElmImb = Parma_GetWeightedEntImbalance(mesh, wtag, mesh->getDimension()); - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); monitorUpdate(maxElmImb, iS, iA); monitorUpdate(avgSides, sS, sA); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) lion_oprint(1, "avgSides %f\n", avgSides); parma::Weights* w = diff --git a/parma/diffMC/parma_ghostMPASWeights.cc b/parma/diffMC/parma_ghostMPASWeights.cc index fd24b9686..78f7f4651 100644 --- a/parma/diffMC/parma_ghostMPASWeights.cc +++ b/parma/diffMC/parma_ghostMPASWeights.cc @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -126,9 +125,9 @@ namespace parma { { GhostElementFinder finder(m, wtag, layers, bridge); findGhostElements(&finder, s); - exchangeGhostElementsFrom(); + exchangeGhostElementsFrom(m->getPCU()); weight += ownedVtxWeight(m, wtag); - exchange(); + exchange(m->getPCU()); } ~GhostMPASWeights() {} double self() { @@ -145,32 +144,32 @@ namespace parma { set(side->first, finder->weight(side->first)); sides->end(); } - void exchangeGhostElementsFrom() { - PCU_Comm_Begin(); + void exchangeGhostElementsFrom(pcu::PCU *PCUObj) { + PCUObj->Begin(); const GhostMPASWeights::Item* ghost; begin(); while( (ghost = iterate()) ) - PCU_COMM_PACK(ghost->first, ghost->second); + PCUObj->Pack(ghost->first, ghost->second); end(); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { + PCUObj->Send(); + while (PCUObj->Listen()) { double ghostsFromPeer = 0; - PCU_COMM_UNPACK(ghostsFromPeer); + PCUObj->Unpack(ghostsFromPeer); weight += ghostsFromPeer; } } - void exchange() { - PCU_Comm_Begin(); + void exchange(pcu::PCU *PCUObj) { + PCUObj->Begin(); const GhostMPASWeights::Item* ghost; begin(); while( (ghost = iterate()) ) - PCU_COMM_PACK(ghost->first, weight); + PCUObj->Pack(ghost->first, weight); end(); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { + PCUObj->Send(); + while (PCUObj->Listen()) { double peerWeight; - PCU_COMM_UNPACK(peerWeight); - int peer = PCU_Comm_Sender(); + PCUObj->Unpack(peerWeight); + int peer = PCUObj->Sender(); set(peer, peerWeight); } } diff --git a/parma/diffMC/parma_ghostOwner.cc b/parma/diffMC/parma_ghostOwner.cc index 1fe51289a..ee1bcb79d 100644 --- a/parma/diffMC/parma_ghostOwner.cc +++ b/parma/diffMC/parma_ghostOwner.cc @@ -1,4 +1,3 @@ -#include #include #include #include "parma_ghostOwner.h" @@ -11,6 +10,6 @@ namespace parma { } bool isOwned(apf::Mesh* m, apf::MeshEntity* v) { - return PCU_Comm_Self() == getOwner(m,v); + return m->getPCU()->Self() == getOwner(m,v); } } diff --git a/parma/diffMC/parma_ghostOwner.h b/parma/diffMC/parma_ghostOwner.h index b770acc10..767e711c7 100644 --- a/parma/diffMC/parma_ghostOwner.h +++ b/parma/diffMC/parma_ghostOwner.h @@ -1,7 +1,6 @@ #ifndef PARMA_GHOSTOWNER_H #define PARMA_GHOSTOWNER_H -#include #include #include diff --git a/parma/diffMC/parma_ghostWeights.cc b/parma/diffMC/parma_ghostWeights.cc index 96c1b1efa..ef78d1bcf 100644 --- a/parma/diffMC/parma_ghostWeights.cc +++ b/parma/diffMC/parma_ghostWeights.cc @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -80,7 +79,7 @@ namespace { current=next; next.clear(); } - PCU_Debug_Print("ghostW peer %d vtx %f edge %f elm %f\n", + m->getPCU()->DebugPrint("ghostW peer %d vtx %f edge %f elm %f\n", peer, weight[0], weight[1], weight[elmDim]); return weight; } @@ -225,9 +224,9 @@ namespace parma { for(int d=dim+1; d<=3; d++) weight[d] = 0; findGhosts(finder, s); - exchangeGhostsFrom(); - exchange(); - PCU_Debug_Print("totW vtx %f edge %f elm %f\n", + exchangeGhostsFrom(m->getPCU()); + exchange(m->getPCU()); + m->getPCU()->DebugPrint("totW vtx %f edge %f elm %f\n", weight[0], weight[1], weight[dim]); } ~GhostWeights() { @@ -251,33 +250,33 @@ namespace parma { set(side->first, finder->weight(side->first)); sides->end(); } - void exchangeGhostsFrom() { - PCU_Comm_Begin(); + void exchangeGhostsFrom(pcu::PCU *PCUObj) { + PCUObj->Begin(); const GhostWeights::Item* ghost; begin(); while( (ghost = iterate()) ) - PCU_Comm_Pack(ghost->first, ghost->second, 4*sizeof(double)); + PCUObj->Pack(ghost->first, ghost->second, 4*sizeof(double)); end(); - PCU_Comm_Send(); + PCUObj->Send(); double ghostsFromPeer[4]; - while (PCU_Comm_Listen()) { - PCU_Comm_Unpack(ghostsFromPeer, 4*sizeof(double)); + while (PCUObj->Listen()) { + PCUObj->Unpack(ghostsFromPeer, 4*sizeof(double)); for(int i=0; i<4; i++) weight[i] += ghostsFromPeer[i]; } } - void exchange() { - PCU_Comm_Begin(); + void exchange(pcu::PCU *PCUObj) { + PCUObj->Begin(); const GhostWeights::Item* ghost; begin(); while( (ghost = iterate()) ) - PCU_Comm_Pack(ghost->first, weight, 4*sizeof(double)); + PCUObj->Pack(ghost->first, weight, 4*sizeof(double)); end(); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { - int peer = PCU_Comm_Sender(); + PCUObj->Send(); + while (PCUObj->Listen()) { + int peer = PCUObj->Sender(); double* peerWeight = get(peer); - PCU_Comm_Unpack(peerWeight, 4*sizeof(double)); + PCUObj->Unpack(peerWeight, 4*sizeof(double)); } } }; diff --git a/parma/diffMC/parma_graphDist.cc b/parma/diffMC/parma_graphDist.cc index f42861d44..d9114257e 100644 --- a/parma/diffMC/parma_graphDist.cc +++ b/parma/diffMC/parma_graphDist.cc @@ -1,5 +1,4 @@ #include -#include #include #include "parma_graphDist.h" #include "parma_dijkstra.h" @@ -63,7 +62,7 @@ namespace { rsum[i] = rsum[i-1] + rmax[i-1] + 1 + maxDistanceIncrease; for(unsigned i=1; igetPCU()->DebugPrint("offset %u is %u\n", i, rsum[i]); // Go backwards so that the largest bdry vtx changes are made first // and won't be augmented in subsequent bdry traversals. @@ -215,7 +214,7 @@ namespace { }; apf::MeshTag* updateDistance(apf::Mesh* m) { - PCU_Debug_Print("updateDistance\n"); + m->getPCU()->DebugPrint("updateDistance\n"); apf::MeshTag* dist = parma::getDistTag(m); parma::DistanceQueue pq(m); getBdryVtx(m,dist,pq); @@ -267,7 +266,7 @@ namespace parma_ordering { if( ! c->has(e) ) continue; cnt++; int d; m->getIntTag(e,dt,&d); - PCU_Debug_Print("cnt %d d %d hasTag %d\n", cnt, d, m->hasTag(e,order)); + m->getPCU()->DebugPrint("cnt %d d %d hasTag %d\n", cnt, d, m->hasTag(e,order)); if( !m->hasTag(e,order) && d > rmax ) { rmax = d; emax = e; @@ -284,14 +283,14 @@ namespace parma_ordering { for(int i=TO_INT(c.size())-1; i>=0; i--) { CompContains* contains = new CompContains(c,i); apf::MeshEntity* src = getMaxDistSeed(m,contains,dist,order); - PCU_Debug_Print("comp %d starting vertex found? %d\n", i, (src != NULL)); + m->getPCU()->DebugPrint("comp %d starting vertex found? %d\n", i, (src != NULL)); start = bfs(m, contains, src, order, start); PCU_ALWAYS_ASSERT(check == c.getIdChecksum()); delete contains; if(start == TO_INT(m->count(0))) { if( i != 0 ) //if not the last component to order parmaCommons::status("%d all vertices visited comp %u of %u\n", - PCU_Comm_Self(), i, c.size()); + m->getPCU()->Self(), i, c.size()); break; } } @@ -342,12 +341,12 @@ namespace parma_ordering { la += abs(vid-uid); } m->end(it); - PCU_Debug_Print("la %d\n", la); - long tot=PCU_Add_Long(TO_LONG(la)); - int max=PCU_Max_Int(la); - int min=PCU_Min_Int(la); - double avg = TO_DOUBLE(tot)/PCU_Comm_Peers(); - if( !PCU_Comm_Self() ) + m->getPCU()->DebugPrint("la %d\n", la); + long tot=m->getPCU()->Add(TO_LONG(la)); + int max=m->getPCU()->Max(la); + int min=m->getPCU()->Min(la); + double avg = TO_DOUBLE(tot)/m->getPCU()->Peers(); + if( !m->getPCU()->Self() ) parmaCommons::status("la min %d max %d avg %.3f\n", min, max, avg); PCU_ALWAYS_ASSERT(check == m->getTagChecksum(order,apf::Mesh::VERTEX)); if( setOrder ) @@ -365,14 +364,14 @@ namespace parma { if( hasDistance(m) ) { t = updateDistance(m); } else { - PCU_Debug_Print("computeDistance\n"); + m->getPCU()->DebugPrint("computeDistance\n"); dcComponents c = dcComponents(m); t = computeDistance(m,c); - if( PCU_Comm_Peers() > 1 && !c.numIso() ) + if( m->getPCU()->Peers() > 1 && !c.numIso() ) if( !hasDistance(m,t) ) { parmaCommons::error("rank %d comp %u iso %u ... " "some vertices don't have distance computed\n", - PCU_Comm_Self(), c.size(), c.numIso()); + m->getPCU()->Self(), c.size(), c.numIso()); PCU_ALWAYS_ASSERT(false); } unsigned* rmax = getMaxDist(m,c,t); @@ -384,17 +383,17 @@ namespace parma { } apf::MeshTag* Parma_BfsReorder(apf::Mesh* m, int) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); PCU_ALWAYS_ASSERT( !hasDistance(m) ); parma::dcComponents c = parma::dcComponents(m); const unsigned checkIds = c.getIdChecksum(); apf::MeshTag* dist = computeDistance(m,c); const unsigned check = m->getTagChecksum(dist,apf::Mesh::VERTEX); - if( PCU_Comm_Peers() > 1 && !c.numIso() ) + if( m->getPCU()->Peers() > 1 && !c.numIso() ) if( !hasDistance(m,dist) ) { parmaCommons::error("rank %d comp %u iso %u ... " "some vertices don't have distance computed\n", - PCU_Comm_Self(), c.size(), c.numIso()); + m->getPCU()->Self(), c.size(), c.numIso()); PCU_ALWAYS_ASSERT(false); } parma_ordering::la(m); @@ -403,6 +402,6 @@ apf::MeshTag* Parma_BfsReorder(apf::Mesh* m, int) { PCU_ALWAYS_ASSERT(checkIds == c.getIdChecksum()); PCU_ALWAYS_ASSERT(check == m->getTagChecksum(dist,apf::Mesh::VERTEX)); m->destroyTag(dist); - parmaCommons::printElapsedTime(__func__,PCU_Time()-t0); + parmaCommons::printElapsedTime(__func__,pcu::Time()-t0,m->getPCU()); return order; } diff --git a/parma/diffMC/parma_ltSelector.cc b/parma/diffMC/parma_ltSelector.cc index 3e99068de..5ab98a9e5 100644 --- a/parma/diffMC/parma_ltSelector.cc +++ b/parma/diffMC/parma_ltSelector.cc @@ -1,6 +1,5 @@ #include #include -#include #include #include "parma_vtxSelector.h" #include "parma_targets.h" @@ -109,16 +108,16 @@ namespace { typedef std::set MigrComm; - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); APF_ITERATE(parma::Mid, sendingEnts, s) - PCU_COMM_PACK(s->first, s->second); - PCU_Comm_Send(); + mesh->getPCU()->Pack(s->first, s->second); + mesh->getPCU()->Send(); MigrComm incoming; //map double w; - while (PCU_Comm_Listen()) { - PCU_COMM_UNPACK(w); - incoming.insert(Migr(PCU_Comm_Sender(),w)); + while (mesh->getPCU()->Listen()) { + mesh->getPCU()->Unpack(w); + incoming.insert(Migr(mesh->getPCU()->Sender(),w)); } double selfW = parma::getWeight(mesh,wtag,primaryDim); @@ -136,15 +135,15 @@ namespace { totW += accept[in->first]; } - PCU_Comm_Begin(); + mesh->getPCU()->Begin(); APF_ITERATE(parma::Mid, accept, a) - PCU_COMM_PACK(a->first, a->second); - PCU_Comm_Send(); + mesh->getPCU()->Pack(a->first, a->second); + mesh->getPCU()->Send(); parma::Mid* capacity = new parma::Mid; double outw; - while (PCU_Comm_Listen()) { - PCU_COMM_UNPACK(outw); - (*capacity)[PCU_Comm_Sender()] = outw; + while (mesh->getPCU()->Listen()) { + mesh->getPCU()->Unpack(outw); + (*capacity)[mesh->getPCU()->Sender()] = outw; } return capacity; } diff --git a/parma/diffMC/parma_preserveTargets.cc b/parma/diffMC/parma_preserveTargets.cc index 387170c48..5214d480c 100644 --- a/parma/diffMC/parma_preserveTargets.cc +++ b/parma/diffMC/parma_preserveTargets.cc @@ -1,4 +1,3 @@ -#include #include "parma_sides.h" #include "parma_weights.h" #include "parma_targets.h" diff --git a/parma/diffMC/parma_shapeOptimizer.cc b/parma/diffMC/parma_shapeOptimizer.cc index b7e8a3dc9..15eb8d8d6 100644 --- a/parma/diffMC/parma_shapeOptimizer.cc +++ b/parma/diffMC/parma_shapeOptimizer.cc @@ -1,4 +1,3 @@ -#include #include #include #include "parma.h" @@ -14,20 +13,20 @@ namespace { using parmaCommons::status; - int getMaxNb(parma::Sides* s) { - return PCU_Max_Int(s->size()); + int getMaxNb(parma::Sides* s, pcu::PCU *PCUObj) { + return PCUObj->Max(s->size()); } class ImbOrMaxNeighbor : public parma::Stop { public: ImbOrMaxNeighbor(parma::Average* nbAvg, double maxNbTol, int targets, int v=0) : nb(nbAvg), nbTol(maxNbTol), tgts(targets), verbose(v) {} - bool stop(double imb, double maxImb) { - int maxTgts = PCU_Max_Int(tgts); + bool stop(double imb, double maxImb, pcu::PCU *PCUObj) { + int maxTgts = PCUObj->Max(tgts); const double nbSlope = nb->avg(); - if( !PCU_Comm_Self() && verbose ) + if( !PCUObj->Self() && verbose ) status("max neighbor slope %f tolerance %f\n", nbSlope, nbTol); - if( !PCU_Comm_Self() && verbose && maxTgts == 0 ) + if( !PCUObj->Self() && verbose && maxTgts == 0 ) status("no targets found... stopping\n"); return imb > maxImb || ( fabs(nbSlope) < nbTol ) || ( maxTgts == 0 ); } @@ -49,9 +48,9 @@ namespace { parma::Weights* w = parma::makeEntWeights(mesh, wtag, s, mesh->getDimension()); parma::Targets* t = - parma::makeShapeTargets(s); + parma::makeShapeTargets(s, mesh->getPCU()); parma::Selector* sel = parma::makeShapeSelector(mesh, wtag); - double maxNb = TO_DOUBLE(getMaxNb(s)); + double maxNb = TO_DOUBLE(getMaxNb(s, mesh->getPCU())); monitorUpdate(maxNb, sS, sA); parma::Stop* stopper = new ImbOrMaxNeighbor(sA, maxNb*.001, t->size(), verbose); diff --git a/parma/diffMC/parma_shapeSelector.cc b/parma/diffMC/parma_shapeSelector.cc index e1d1af4e0..967a9f5a3 100644 --- a/parma/diffMC/parma_shapeSelector.cc +++ b/parma/diffMC/parma_shapeSelector.cc @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/parma/diffMC/parma_shapeTargets.cc b/parma/diffMC/parma_shapeTargets.cc index be9bcbc81..7fd3baa1c 100644 --- a/parma/diffMC/parma_shapeTargets.cc +++ b/parma/diffMC/parma_shapeTargets.cc @@ -1,4 +1,3 @@ -#include #include #include "parma_sides.h" #include "parma_weights.h" @@ -14,9 +13,9 @@ namespace parma { class ShapeTargets : public Targets { public: - ShapeTargets(Sides* s) { + ShapeTargets(Sides* s, pcu::PCU *PCUObj) { smallLimit = 10; - init(s); + init(s, PCUObj); totW = 0; } double total() { @@ -26,12 +25,12 @@ namespace parma { ShapeTargets(); int smallLimit; double totW; - void init(Sides* s) { - const unsigned maxNb = TO_UINT(PCU_Max_Int(s->size())); + void init(Sides* s, pcu::PCU *PCUObj) { + const unsigned maxNb = TO_UINT(PCUObj->Max(s->size())); if( s->size() != maxNb ) return; - PCU_Debug_Print("maxNb %d\n", maxNb); + PCUObj->DebugPrint("maxNb %d\n", maxNb); std::string sstr = s->print("sides"); - PCU_Debug_Print("%s\n", sstr.c_str()); + PCUObj->DebugPrint("%s\n", sstr.c_str()); int small = INT_MAX; s->begin(); const Sides::Item* side; @@ -39,7 +38,7 @@ namespace parma { if( side->second < small ) small = side->second; s->end(); - PCU_Debug_Print("small %d\n", small); + PCUObj->DebugPrint("small %d\n", small); if( small > smallLimit ) return; s->begin(); while( (side = s->iterate()) ) @@ -47,10 +46,10 @@ namespace parma { set(side->first, small); s->end(); std::string tgtstr = print("targets"); - PCU_Debug_Print("%s\n", tgtstr.c_str()); + PCUObj->DebugPrint("%s\n", tgtstr.c_str()); } }; - Targets* makeShapeTargets(Sides* s) { - return new ShapeTargets(s); + Targets* makeShapeTargets(Sides* s, pcu::PCU *PCUObj) { + return new ShapeTargets(s, PCUObj); } } //end namespace diff --git a/parma/diffMC/parma_sides.cc b/parma/diffMC/parma_sides.cc index 68404a5bf..62d6b8bf0 100644 --- a/parma/diffMC/parma_sides.cc +++ b/parma/diffMC/parma_sides.cc @@ -1,8 +1,7 @@ -#include #include "parma_sides.h" #include "parma_convert.h" -double parma::avgSharedSides(parma::Sides* s) { +double parma::avgSharedSides(parma::Sides* s, pcu::PCU *PCUObj) { double tot = TO_DOUBLE(s->total()); - tot = PCU_Add_Double(tot); - return tot / PCU_Comm_Peers(); + tot = PCUObj->Add(tot); + return tot / PCUObj->Peers(); } diff --git a/parma/diffMC/parma_sides.h b/parma/diffMC/parma_sides.h index cf1ba2557..b22632a8d 100644 --- a/parma/diffMC/parma_sides.h +++ b/parma/diffMC/parma_sides.h @@ -16,7 +16,7 @@ namespace parma { Sides* makeElmSideSides(apf::Mesh* m); Sides* makeVtxSides(apf::Mesh* m); - double avgSharedSides(Sides* s); + double avgSharedSides(Sides* s, pcu::PCU *PCUObj); } #endif diff --git a/parma/diffMC/parma_step.cc b/parma/diffMC/parma_step.cc index a30cc7d70..9e1b5c49e 100644 --- a/parma/diffMC/parma_step.cc +++ b/parma/diffMC/parma_step.cc @@ -1,4 +1,3 @@ -#include #include #include "parma_step.h" #include "parma_sides.h" @@ -29,17 +28,17 @@ namespace parma { bool Stepper::step(double maxImb, int verbosity) { double imb, avg; - getImbalance(weights, imb, avg); - if ( !PCU_Comm_Self() && verbosity ) + getImbalance(weights, imb, avg, m->getPCU()); + if ( !m->getPCU()->Self() && verbosity ) status("%s imbalance %.3f avg %.3f\n", name, imb, avg); - if ( stop->stop(imb,maxImb) ) + if ( stop->stop(imb,maxImb,m->getPCU()) ) return false; apf::Migration* plan = selects->run(targets); - int planSz = PCU_Add_Int(plan->count()); - const double t0 = PCU_Time(); + int planSz = m->getPCU()->Add(plan->count()); + const double t0 = pcu::Time(); m->migrate(plan); - if ( !PCU_Comm_Self() && verbosity ) - status("%d elements migrated in %f seconds\n", planSz, PCU_Time()-t0); + if ( !m->getPCU()->Self() && verbosity ) + status("%d elements migrated in %f seconds\n", planSz, pcu::Time()-t0); if( verbosity > 1 ) Parma_PrintPtnStats(m, "endStep", (verbosity>2)); return true; diff --git a/parma/diffMC/parma_stop.cc b/parma/diffMC/parma_stop.cc index b110d340a..08b48b346 100644 --- a/parma/diffMC/parma_stop.cc +++ b/parma/diffMC/parma_stop.cc @@ -1,16 +1,15 @@ #include "parma_stop.h" #include "parma_commons.h" -#include #include namespace parma { BalOrStall::BalOrStall(Average* imb, Average* sides, double sidesTol, int v) : i(imb), s(sides), sTol(sidesTol), verbose(v) {} - bool BalOrStall::stop(double imb, double maxImb) { + bool BalOrStall::stop(double imb, double maxImb, pcu::PCU *PCUObj) { const double iTol = (maxImb-1)*.01; const double iSlope = i->avg(); const double sSlope = s->avg(); - if( !PCU_Comm_Self() && verbose ) + if( !PCUObj->Self() && verbose ) parmaCommons::status("imbSlope %f sidesSlope %f\n", iSlope, sSlope); return imb < maxImb || ( fabs(iSlope) < iTol && fabs(sSlope) < sTol ); diff --git a/parma/diffMC/parma_stop.h b/parma/diffMC/parma_stop.h index f392ebfe3..ce7513126 100644 --- a/parma/diffMC/parma_stop.h +++ b/parma/diffMC/parma_stop.h @@ -2,17 +2,18 @@ #define PARMA_STOP_H #include "parma_monitor.h" +#include "PCU.h" namespace parma { class Stop { public: virtual ~Stop() {} - virtual bool stop(double imb, double maxImb)=0; + virtual bool stop(double imb, double maxImb, pcu::PCU*)=0; }; class Less : public Stop { public: ~Less() {} - bool stop(double imb, double maxImb) { + bool stop(double imb, double maxImb, pcu::PCU*) { return imb < maxImb; } }; @@ -20,7 +21,7 @@ namespace parma { public: BalOrStall(Average* imb, Average* sides, double sidesTol, int verbose=0); ~BalOrStall() {} - bool stop(double imb, double maxImb); + bool stop(double imb, double maxImb, pcu::PCU *PCUObj); private: Average* i; Average* s; diff --git a/parma/diffMC/parma_targets.h b/parma/diffMC/parma_targets.h index 93fd95f07..38f82f61d 100644 --- a/parma/diffMC/parma_targets.h +++ b/parma/diffMC/parma_targets.h @@ -22,7 +22,7 @@ namespace parma { double vtxTol, double alpha); Targets* makeElmLtVtxEdgeTargets(Sides* s, Weights* w[3], int sideTol, double vtxTol, double edgeTol, double alpha); - Targets* makeShapeTargets(Sides* s); + Targets* makeShapeTargets(Sides* s, pcu::PCU *PCUObj); Targets* makeGhostTargets(Sides* s, Weights* w, Ghosts* g, double alpha); } #endif diff --git a/parma/diffMC/parma_vtxBalancer.cc b/parma/diffMC/parma_vtxBalancer.cc index 43edeae28..ec2525222 100644 --- a/parma/diffMC/parma_vtxBalancer.cc +++ b/parma/diffMC/parma_vtxBalancer.cc @@ -1,4 +1,3 @@ -#include #include "parma.h" #include "parma_step.h" #include "parma_balancer.h" @@ -22,9 +21,9 @@ namespace { VtxBalancer(apf::Mesh* m, double f, int v) : Balancer(m, f, v, "vertices") { parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = TO_INT(parma::avgSharedSides(s)); + sideTol = TO_INT(parma::avgSharedSides(s, mesh->getPCU())); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !m->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } @@ -36,10 +35,10 @@ namespace { parma::Targets* t = parma::makeWeightSideTargets(s, w, sideTol, factor); parma::Selector* sel = parma::makeVtxSelector(mesh, wtag); - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); monitorUpdate(maxVtxImb, iS, iA); monitorUpdate(avgSides, sS, sA); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("vtxImb %f avgSides %f\n", maxVtxImb, avgSides); parma::BalOrStall* stopper = new parma::BalOrStall(iA, sA, sideTol*.001, verbose); @@ -51,7 +50,7 @@ namespace { apf::Balancer* Parma_MakeVtxBalancer(apf::Mesh* m, double stepFactor, int verbosity) { - if( !PCU_Comm_Self() && verbosity ) + if( !m->getPCU()->Self() && verbosity ) status("stepFactor %.3f\n", stepFactor); return new VtxBalancer(m, stepFactor, verbosity); } diff --git a/parma/diffMC/parma_vtxEdgeElmBalancer.cc b/parma/diffMC/parma_vtxEdgeElmBalancer.cc index 83a1a3563..02b795c8c 100644 --- a/parma/diffMC/parma_vtxEdgeElmBalancer.cc +++ b/parma/diffMC/parma_vtxEdgeElmBalancer.cc @@ -1,4 +1,3 @@ -#include #include #include #include "parma_balancer.h" @@ -21,25 +20,25 @@ namespace { VtxEdgeBalancer(apf::Mesh* m, double f, double maxV, int v) : Balancer(m, f, v, "edges") { maxVtx = maxV; - if( !PCU_Comm_Self() && verbose ) { + if( !m->getPCU()->Self() && verbose ) { status("stepFactor %.3f\n", f); status("maxVtx %.3f\n", maxVtx); } parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = TO_INT(parma::avgSharedSides(s)); + sideTol = TO_INT(parma::avgSharedSides(s, mesh->getPCU())); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !m->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { const double maxVtxImb = Parma_GetWeightedEntImbalance(mesh, wtag, 0); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("vtx imbalance %.3f\n", maxVtxImb); const double maxEdgeImb = Parma_GetWeightedEntImbalance(mesh, wtag, 1); parma::Sides* s = parma::makeVtxSides(mesh); - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); parma::Weights* w[2] = {parma::makeEntWeights(mesh, wtag, s, 0), parma::makeEntWeights(mesh, wtag, s, 1)}; @@ -49,7 +48,7 @@ namespace { monitorUpdate(maxEdgeImb, iS, iA); monitorUpdate(avgSides, sS, sA); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("edgeImb %f avgSides %f\n", maxEdgeImb, avgSides); parma::BalOrStall* stopper = new parma::BalOrStall(iA, sA, sideTol*.001, verbose); diff --git a/parma/diffMC/parma_vtxEdgeTargets.cc b/parma/diffMC/parma_vtxEdgeTargets.cc index ebc51aded..177ddeba8 100644 --- a/parma/diffMC/parma_vtxEdgeTargets.cc +++ b/parma/diffMC/parma_vtxEdgeTargets.cc @@ -1,4 +1,3 @@ -#include #include "parma_sides.h" #include "parma_weights.h" #include "parma_targets.h" diff --git a/parma/diffMC/parma_vtxElmBalancer.cc b/parma/diffMC/parma_vtxElmBalancer.cc index 14362e09f..90a96b14d 100644 --- a/parma/diffMC/parma_vtxElmBalancer.cc +++ b/parma/diffMC/parma_vtxElmBalancer.cc @@ -1,4 +1,3 @@ -#include #include #include #include "parma_balancer.h" @@ -22,14 +21,14 @@ namespace { ElmLtVtx(apf::Mesh* m, double f, double maxV, int v) : Balancer(m, f, v, "elements") { maxVtx = maxV; - if( !PCU_Comm_Self() && verbose ) { + if( !m->getPCU()->Self() && verbose ) { status("stepFactor %.3f\n", f); status("maxVtx %.3f\n", maxVtx); } parma::Sides* s = parma::makeVtxSides(mesh); - sideTol = TO_INT(parma::avgSharedSides(s)); + sideTol = TO_INT(parma::avgSharedSides(s, mesh->getPCU())); delete s; - if( !PCU_Comm_Self() && verbose ) + if( !m->getPCU()->Self() && verbose ) status("sideTol %d\n", sideTol); } bool runStep(apf::MeshTag* wtag, double tolerance) { @@ -37,7 +36,7 @@ namespace { Parma_GetWeightedEntImbalance(mesh, wtag, 0); const double maxElmImb = Parma_GetWeightedEntImbalance(mesh, wtag, mesh->getDimension()); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("vtx imbalance %.3f\n", maxVtxImb); parma::Sides* s = parma::makeVtxSides(mesh); parma::Weights* vtxW = parma::makeEntWeights(mesh, wtag, s, 0); @@ -49,10 +48,10 @@ namespace { parma::Selector* sel = parma::makeElmLtVtxSelector(mesh, wtag, maxVtx); - double avgSides = parma::avgSharedSides(s); + double avgSides = parma::avgSharedSides(s, mesh->getPCU()); monitorUpdate(maxElmImb, iS, iA); monitorUpdate(avgSides, sS, sA); - if( !PCU_Comm_Self() && verbose ) + if( !mesh->getPCU()->Self() && verbose ) status("elmImb %f avgSides %f\n", maxElmImb, avgSides); parma::BalOrStall* stopper = new parma::BalOrStall(iA, sA, sideTol*.001, verbose); diff --git a/parma/diffMC/parma_vtxPtnWriter.cc b/parma/diffMC/parma_vtxPtnWriter.cc index 5e038081a..62763b551 100644 --- a/parma/diffMC/parma_vtxPtnWriter.cc +++ b/parma/diffMC/parma_vtxPtnWriter.cc @@ -21,16 +21,16 @@ namespace { } public: Ptn(apf::Mesh* m) { - const long totv = PCU_Add_Long(countOwned(m)); - c = pp = totv / PCU_Comm_Peers(); - f = pp * PCU_Comm_Self(); - const int remainder = totv % PCU_Comm_Peers(); - if( PCU_Comm_Self() == PCU_Comm_Peers()-1 ) + const long totv = m->getPCU()->Add(countOwned(m)); + c = pp = totv / m->getPCU()->Peers(); + f = pp * m->getPCU()->Self(); + const int remainder = totv % m->getPCU()->Peers(); + if( m->getPCU()->Self() == m->getPCU()->Peers()-1 ) c += remainder; } - int getWriter(int id) { + int getWriter(int id, pcu::PCU *PCUObj) { int writer = id / pp; - if ( writer == PCU_Comm_Peers() ) + if ( writer == PCUObj->Peers() ) writer--; return writer; } @@ -48,21 +48,21 @@ namespace { apf::MeshEntity* vtx; apf::MeshIterator* itr = m->begin(0); int id = 0; - PCU_Comm_Begin(); + m->getPCU()->Begin(); while( (vtx = m->iterate(itr)) ) { if( parma::isOwned(m, vtx) ) { m->getIntTag(vtx, t, &id); - PCU_COMM_PACK(p.getWriter(id), id); + m->getPCU()->Pack(p.getWriter(id, m->getPCU()), id); } } m->end(itr); - PCU_Comm_Send(); - while( PCU_Comm_Receive() ) { + m->getPCU()->Send(); + while( m->getPCU()->Receive() ) { int id = 0; - PCU_COMM_UNPACK(id); + m->getPCU()->Unpack(id); const int idx = id - p.first(); PCU_ALWAYS_ASSERT(idx >= 0 && idx < p.count()); - ptn[idx] = PCU_Comm_Sender(); + ptn[idx] = m->getPCU()->Sender(); } } @@ -71,9 +71,9 @@ namespace { f << a[i] << '\n'; } - void open(const char* name, std::fstream& f) { + void open(const char* name, std::fstream& f, pcu::PCU *PCUObj) { std::stringstream ss; - ss << name << PCU_Comm_Self() << ".ptn"; + ss << name << PCUObj->Self() << ".ptn"; std::string s = ss.str(); f.open(s.c_str(), std::fstream::out); } @@ -81,7 +81,7 @@ namespace { void writeVtxPtn(apf::Mesh* m, const char* name) { PCU_ALWAYS_ASSERT(name); std::fstream f; - open(name,f); + open(name,f,m->getPCU()); Ptn p(m); int* ptn = new int[p.count()]; getPtnArray(m, p, ptn); diff --git a/parma/diffMC/parma_vtxSelector.cc b/parma/diffMC/parma_vtxSelector.cc index 587012990..d28197e24 100644 --- a/parma/diffMC/parma_vtxSelector.cc +++ b/parma/diffMC/parma_vtxSelector.cc @@ -1,4 +1,3 @@ -#include #include #include "parma.h" #include "parma_vtxSelector.h" @@ -158,7 +157,7 @@ namespace parma { } destroy(peers); } - PCU_Debug_Print("sent %u disconnected cavities\n", dcCnt); + mesh->getPCU()->DebugPrint("sent %u disconnected cavities\n", dcCnt); delete bdryVerts; return planW; } diff --git a/parma/diffMC/parma_weightSideTargets.cc b/parma/diffMC/parma_weightSideTargets.cc index f721cf116..aa8e9c938 100644 --- a/parma/diffMC/parma_weightSideTargets.cc +++ b/parma/diffMC/parma_weightSideTargets.cc @@ -1,4 +1,3 @@ -#include #include "parma_sides.h" #include "parma_weights.h" #include "parma_targets.h" diff --git a/parma/diffMC/parma_weights.h b/parma/diffMC/parma_weights.h index be028235d..40ddd299d 100644 --- a/parma/diffMC/parma_weights.h +++ b/parma/diffMC/parma_weights.h @@ -26,8 +26,8 @@ namespace parma { double getMaxWeight(apf::Mesh* m, apf::MeshTag* w, int entDim); double getAvgWeight(apf::Mesh* m, apf::MeshTag* w, int entDim); double getWeight(apf::Mesh* m, apf::MeshTag* w, int entDim); - double getMaxWeight(Weights* w); - void getImbalance(Weights* w, double& imb, double& avg); + double getMaxWeight(Weights* w, pcu::PCU *PCUObj); + void getImbalance(Weights* w, double& imb, double& avg, pcu::PCU *PCUObj); } #endif diff --git a/parma/group/parma_group.cc b/parma/group/parma_group.cc index 2afec2417..799428082 100644 --- a/parma/group/parma_group.cc +++ b/parma/group/parma_group.cc @@ -1,11 +1,12 @@ -#include #include +#include +#include using apf::Remap; static void retreat(apf::Mesh2* m, Remap& remap) { - int to = remap(PCU_Comm_Self()); + int to = remap(m->getPCU()->Self()); apf::Migration* plan = new apf::Migration(m); apf::MeshIterator* it = m->begin(m->getDimension()); apf::MeshEntity* e; @@ -27,25 +28,41 @@ typedef Parma_GroupCode GroupCode; static void runInGroups( apf::Mesh2* m, + pcu::PCU *PCUObj, Remap& inMap, Remap& groupMap, Remap& outMap, - GroupCode& code) + GroupCode& code + ) { - int self = PCU_Comm_Self(); + int self; + pcu::PCU* expandedPCU; + if(PCUObj == nullptr){ + self = m->getPCU()->Self(); + expandedPCU = m->getPCU(); + } else { + self = PCUObj->Self(); + expandedPCU = PCUObj; + } + int groupRank = inMap(self); int group = groupMap(self); - MPI_Comm oldComm = PCU_Get_Comm(); + MPI_Comm groupComm; - MPI_Comm_split(oldComm, group, groupRank, &groupComm); - PCU_Switch_Comm(groupComm); - if (m) + MPI_Comm_split(expandedPCU->GetMPIComm(), group, groupRank, &groupComm); + auto groupedPCU = std::unique_ptr(new pcu::PCU(groupComm)); + if (m){ + m->switchPCU(groupedPCU.get()); apf::remapPartition(m, inMap); + } + code.PCUObj = std::move(groupedPCU); code.run(group); - PCU_Switch_Comm(oldComm); + MPI_Comm_free(&groupComm); - if (m) + if (m){ + m->switchPCU(expandedPCU); apf::remapPartition(m, outMap); + } } struct RetreatCode : public GroupCode @@ -79,28 +96,46 @@ static void retreatToGroup( Remap& retreatMap, Remap& groupMap, Remap& outMap, - GroupCode& code) + GroupCode& code, + pcu::PCU *PCUObj) { retreat(m, retreatMap); RetreatCode retreatCode(code, m, factor); - runInGroups(m, inMap, groupMap, outMap, retreatCode); + runInGroups(m, PCUObj, inMap, groupMap, outMap, retreatCode); m->migrate(retreatCode.plan); } -void Parma_ShrinkPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun) +void Parma_ShrinkPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun, pcu::PCU *PCUObj) { + if(m == nullptr){ + PCU_ALWAYS_ASSERT(PCUObj != nullptr); + } + if(m != nullptr && PCUObj != nullptr){ + PCU_ALWAYS_ASSERT(m->getPCU() == PCUObj); + } apf::Divide inMap(factor); apf::Modulo groupMap(factor); apf::Round retreatMap(factor); apf::Multiply outMap(factor); - retreatToGroup(m, factor, inMap, retreatMap, groupMap, outMap, toRun); + retreatToGroup(m, factor, inMap, retreatMap, groupMap, outMap, toRun, PCUObj); } -void Parma_SplitPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun) +void Parma_SplitPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun, pcu::PCU *PCUObj) { + if(m == nullptr){ + PCU_ALWAYS_ASSERT(PCUObj != nullptr); + } + if(m != nullptr && PCUObj != nullptr){ + PCU_ALWAYS_ASSERT(m->getPCU() == PCUObj); + } apf::Modulo inMap(factor); apf::Divide groupMap(factor); - apf::Unmodulo outMap(PCU_Comm_Self(), factor); - runInGroups(m, inMap, groupMap, outMap, toRun); + if(PCUObj == nullptr){ + apf::Unmodulo outMap(m->getPCU()->Self(), factor); + runInGroups(m, PCUObj, inMap, groupMap, outMap, toRun); + } else { + apf::Unmodulo outMap(PCUObj->Self(), factor); + runInGroups(m, PCUObj, inMap, groupMap, outMap, toRun); + } } diff --git a/parma/parma.cc b/parma/parma.cc index 56d57bec7..636a1ffc5 100644 --- a/parma/parma.cc +++ b/parma/parma.cc @@ -1,4 +1,3 @@ -#include #include #include "parma.h" #include "diffMC/maximalIndependentSet/mis.h" @@ -89,24 +88,24 @@ namespace { } void getWeightedStats( - int dim, double (*loc)[4], double (*tot)[4], + pcu::PCU *PCUObj, int dim, double (*loc)[4], double (*tot)[4], double (*min)[4], double (*max)[4], double (*avg)[4]) { for(int d=0; d<=dim; d++) (*min)[d] = (*max)[d] = (*tot)[d] = (*loc)[d]; - PCU_Min_Doubles(*min, dim+1); - PCU_Max_Doubles(*max, dim+1); - PCU_Add_Doubles(*tot, dim+1); + PCUObj->Min(*min, dim+1); + PCUObj->Max(*max, dim+1); + PCUObj->Add(*tot, dim+1); for(int d=0; d<=dim; d++) { (*avg)[d] = (*tot)[d]; - (*avg)[d] /= TO_DOUBLE(PCU_Comm_Peers()); + (*avg)[d] /= TO_DOUBLE(PCUObj->Peers()); } } - void getStats(int& loc, long& tot, int& min, int& max, double& avg) { - min = PCU_Min_Int(loc); - max = PCU_Max_Int(loc); - tot = PCU_Add_Long(TO_LONG(loc)); - avg = TO_DOUBLE(tot) / PCU_Comm_Peers(); + void getStats(pcu::PCU *PCUObj, int& loc, long& tot, int& min, int& max, double& avg) { + min = PCUObj->Min(loc); + max = PCUObj->Max(loc); + tot = PCUObj->Add(TO_LONG(loc)); + avg = TO_DOUBLE(tot) / PCUObj->Peers(); } using parmaCommons::status; @@ -120,7 +119,7 @@ namespace { for(int d=0; d<=dim; d++) ss << entNames[d] << ' '; ss << "dc nb owned_bdry shared_bdry model_bdry shSidesToElm > " - << PCU_Comm_Self()+1 << ' '; + << m->getPCU()->Self()+1 << ' '; for(int d=0; d<=dim; d++) ss << m->count(d) << ' '; ss << m->count(m->getDimension()) << ' ' @@ -130,7 +129,7 @@ namespace { << surf/TO_DOUBLE(vol); std::string s = ss.str(); lion_eprint(1, "%s\n", s.c_str()); - PCU_Barrier(); + m->getPCU()->Barrier(); } void writeWeightedEntStats(apf::Mesh* m, apf::MeshTag* w, std::string key) { @@ -138,9 +137,9 @@ namespace { getPartWeights(m, w, &weight); double minEnt[4] = {0,0,0,0}, maxEnt[4] = {0,0,0,0}; double totEnt[4] = {0,0,0,0}, avgEnt[4] = {0,0,0,0}; - getWeightedStats(m->getDimension(), &weight, &totEnt, &minEnt, &maxEnt, &avgEnt); + getWeightedStats(m->getPCU(), m->getDimension(), &weight, &totEnt, &minEnt, &maxEnt, &avgEnt); const char* orders[4] = {"vtx","edge","face","rgn"}; - if(!PCU_Comm_Self()) { + if(!m->getPCU()->Self()) { for( int d=0; d<=m->getDimension(); d++) status("%s weighted %s " "%.1f %.1f %.1f %.3f\n", @@ -168,10 +167,10 @@ void Parma_GetEntImbalance(apf::Mesh* mesh, double (*entImb)[4]) { dims = TO_SIZET(mesh->getDimension()) + 1; for(size_t i=0; i < dims; i++) tot[i] = (*entImb)[i] = mesh->count(TO_INT(i)); - PCU_Add_Doubles(tot, dims); - PCU_Max_Doubles(*entImb, dims); + mesh->getPCU()->Add(tot, dims); + mesh->getPCU()->Max(*entImb, dims); for(size_t i=0; i < dims; i++) - (*entImb)[i] /= (tot[i]/PCU_Comm_Peers()); + (*entImb)[i] /= (tot[i]/mesh->getPCU()->Peers()); for(size_t i=dims; i < 4; i++) (*entImb)[i] = 1.0; } @@ -183,10 +182,10 @@ void Parma_GetWeightedEntImbalance(apf::Mesh* mesh, apf::MeshTag* w, double tot[4] = {0,0,0,0}; for(size_t i=0; i < dims; i++) tot[i] = (*entImb)[i]; - PCU_Add_Doubles(tot, TO_SIZET(dims)); - PCU_Max_Doubles(*entImb, TO_SIZET(dims)); + mesh->getPCU()->Add(tot, TO_SIZET(dims)); + mesh->getPCU()->Max(*entImb, TO_SIZET(dims)); for(size_t i=0; i < dims; i++) - (*entImb)[i] /= (tot[i]/PCU_Comm_Peers()); + (*entImb)[i] /= (tot[i]/mesh->getPCU()->Peers()); for(size_t i=dims; i < 4; i++) (*entImb)[i] = 1.0; } @@ -200,9 +199,9 @@ double Parma_GetWeightedEntImbalance(apf::Mesh* m, apf::MeshTag* w, while ((e = m->iterate(it))) sum += getEntWeight(m, e, w); m->end(it); - double tot = PCU_Add_Double(sum); - double max = PCU_Max_Double(sum); - return max/(tot/PCU_Comm_Peers()); + double tot = m->getPCU()->Add(sum); + double max = m->getPCU()->Max(sum); + return max/(tot/m->getPCU()->Peers()); } void Parma_GetNeighborStats(apf::Mesh* m, int& max, int& numMaxParts, @@ -210,9 +209,9 @@ void Parma_GetNeighborStats(apf::Mesh* m, int& max, int& numMaxParts, mii nborToShared; getNeighborCounts(m,nborToShared); loc = TO_INT(nborToShared.size())-1; - max = PCU_Max_Int(loc); - avg = TO_DOUBLE(PCU_Add_Int(loc)) / PCU_Comm_Peers(); - numMaxParts = PCU_Add_Int( (loc==max) ); + max = m->getPCU()->Max(loc); + avg = TO_DOUBLE(m->getPCU()->Add(loc)) / m->getPCU()->Peers(); + numMaxParts = m->getPCU()->Add( (loc==max) ); } void Parma_WriteSmallNeighbors(apf::Mesh* m, int small, const char* prefix) { @@ -224,8 +223,8 @@ void Parma_WriteSmallNeighbors(apf::Mesh* m, int small, const char* prefix) { for(int i=0; isecond == i+1 ) smallCnt[i]++; - PCU_Add_Ints(smallCnt,small); - if( !PCU_Comm_Self() ) { + m->getPCU()->Add(smallCnt,small); + if( !m->getPCU()->Self() ) { std::stringstream ss; for(int i=0; igetPCU()->Max(loc); int smallest = INT_MAX; if( loc == max ) { APF_ITERATE(mii, nborToShared, nbor) if( nbor->second < smallest ) smallest = nbor->second; } - return PCU_Min_Int(smallest); + return m->getPCU()->Min(smallest); } void Parma_GetOwnedBdryVtxStats(apf::Mesh* m, int& loc, long& tot, int& min, int& max, double& avg) { loc = numBdryVtx(m); - getStats(loc, tot, min, max, avg); + getStats(m->getPCU(), loc, tot, min, max, avg); } void Parma_GetSharedBdryVtxStats(apf::Mesh* m, int& loc, long& tot, int& min, int& max, double& avg) { bool onlyShared = true; loc = numBdryVtx(m,onlyShared); - getStats(loc, tot, min, max, avg); + getStats(m->getPCU(), loc, tot, min, max, avg); } void Parma_GetMdlBdryVtxStats(apf::Mesh* m, int& loc, long& tot, int& min, int& max, double& avg) { loc = numMdlBdryVtx(m); - getStats(loc, tot, min, max, avg); + getStats(m->getPCU(), loc, tot, min, max, avg); } void Parma_GetDisconnectedStats(apf::Mesh* m, int& max, double& avg, int& loc) { dcPart dc(m); loc = TO_INT(dc.getNumDcComps()); - max = PCU_Max_Int(loc); - avg = TO_DOUBLE( PCU_Add_Int(loc) ) / PCU_Comm_Peers(); + max = m->getPCU()->Max(loc); + avg = TO_DOUBLE( m->getPCU()->Add(loc) ) / m->getPCU()->Peers(); } void Parma_ProcessDisconnectedParts(apf::Mesh* m) { @@ -297,24 +296,24 @@ void Parma_PrintPtnStats(apf::Mesh* m, std::string key, bool fine) { } void Parma_PrintWeightedPtnStats(apf::Mesh* m, apf::MeshTag* w, std::string key, bool fine) { - PCU_Debug_Print("%s vtx %lu\n", key.c_str(), m->count(0)); - PCU_Debug_Print("%s edge %lu\n", key.c_str(), m->count(1)); - PCU_Debug_Print("%s face %lu\n", key.c_str(), m->count(2)); + m->getPCU()->DebugPrint("%s vtx %lu\n", key.c_str(), m->count(0)); + m->getPCU()->DebugPrint("%s edge %lu\n", key.c_str(), m->count(1)); + m->getPCU()->DebugPrint("%s face %lu\n", key.c_str(), m->count(2)); if( m->getDimension() == 3 ) - PCU_Debug_Print("%s rgn %lu\n", key.c_str(), m->count(3)); + m->getPCU()->DebugPrint("%s rgn %lu\n", key.c_str(), m->count(3)); int maxDc = 0; double avgDc = 0; int locDc = 0; Parma_GetDisconnectedStats(m, maxDc, avgDc, locDc); - PCU_Debug_Print("%s dc %d\n", key.c_str(), locDc); + m->getPCU()->DebugPrint("%s dc %d\n", key.c_str(), locDc); int maxNb = 0, maxNbParts = 0; double avgNb = 0; int locNb = 0; Parma_GetNeighborStats(m, maxNb, maxNbParts, avgNb, locNb); int smallSideMaxNbPart = Parma_GetSmallestSideMaxNeighborParts(m); - PCU_Debug_Print("%s neighbors %d\n", key.c_str(), locNb); + m->getPCU()->DebugPrint("%s neighbors %d\n", key.c_str(), locNb); int locV[3], minV[3], maxV[3]; long totV[3]; @@ -322,20 +321,20 @@ void Parma_PrintWeightedPtnStats(apf::Mesh* m, apf::MeshTag* w, std::string key, Parma_GetOwnedBdryVtxStats(m, locV[0], totV[0], minV[0], maxV[0], avgV[0]); Parma_GetSharedBdryVtxStats(m, locV[1], totV[1], minV[1], maxV[1], avgV[1]); Parma_GetMdlBdryVtxStats(m, locV[2], totV[2], minV[2], maxV[2], avgV[2]); - PCU_Debug_Print("%s ownedBdryVtx %d\n", key.c_str(), locV[0]); - PCU_Debug_Print("%s sharedBdryVtx %d\n", key.c_str(), locV[1]); - PCU_Debug_Print("%s mdlBdryVtx %d\n", key.c_str(), locV[2]); + m->getPCU()->DebugPrint("%s ownedBdryVtx %d\n", key.c_str(), locV[0]); + m->getPCU()->DebugPrint("%s sharedBdryVtx %d\n", key.c_str(), locV[1]); + m->getPCU()->DebugPrint("%s mdlBdryVtx %d\n", key.c_str(), locV[2]); int surf = numSharedSides(m); double vol = TO_DOUBLE( m->count(m->getDimension()) ); double surfToVol = surf/vol; - double minSurfToVol = PCU_Min_Double(surfToVol); - double maxSurfToVol = PCU_Max_Double(surfToVol); - double avgSurfToVol = PCU_Add_Double(surfToVol) / PCU_Comm_Peers(); - PCU_Debug_Print("%s sharedSidesToElements %.3f\n", key.c_str(), surfToVol); + double minSurfToVol = m->getPCU()->Min(surfToVol); + double maxSurfToVol = m->getPCU()->Max(surfToVol); + double avgSurfToVol = m->getPCU()->Add(surfToVol) / m->getPCU()->Peers(); + m->getPCU()->DebugPrint("%s sharedSidesToElements %.3f\n", key.c_str(), surfToVol); int empty = (m->count(m->getDimension()) == 0 ) ? 1 : 0; - empty = PCU_Add_Int(empty); + empty = m->getPCU()->Add(empty); double imb[4] = {0, 0, 0, 0}; Parma_GetWeightedEntImbalance(m,w,&imb); @@ -343,14 +342,14 @@ void Parma_PrintWeightedPtnStats(apf::Mesh* m, apf::MeshTag* w, std::string key, if (fine) writeFineStats(m, key, locDc, locNb, locV, surf, vol); - PCU_Debug_Print("%s vtxAdjacentNeighbors ", key.c_str()); + m->getPCU()->DebugPrint("%s vtxAdjacentNeighbors ", key.c_str()); apf::Parts peers; apf::getPeers(m,0,peers); APF_ITERATE(apf::Parts,peers,p) - PCU_Debug_Print("%d ", *p); - PCU_Debug_Print("\n"); + m->getPCU()->DebugPrint("%d ", *p); + m->getPCU()->DebugPrint("\n"); - if( 0 == PCU_Comm_Self() ) { + if( 0 == m->getPCU()->Self() ) { status("%s disconnected %d %.3f\n", key.c_str(), maxDc, avgDc); status("%s neighbors %d %.3f\n", @@ -365,7 +364,7 @@ void Parma_PrintWeightedPtnStats(apf::Mesh* m, apf::MeshTag* w, std::string key, const int smallestSide = 10; Parma_WriteSmallNeighbors(m, smallestSide, key.c_str()); writeWeightedEntStats(m,w,key); - if( 0 == PCU_Comm_Self() ) { + if( 0 == m->getPCU()->Self() ) { status("%s owned bdry vtx " "%ld %d %d %.3f\n", key.c_str(), totV[0], maxV[0], minV[0], avgV[0]); @@ -412,15 +411,15 @@ int Parma_MisNumbering(apf::Mesh* m, int d) { int misNumber=-1; int iter=0; int misSize=0; - while( misSize != PCU_Comm_Peers() ) { - if( mis(part, false, true) || 1 == part.net.size() ) { + while( misSize != m->getPCU()->Peers() ) { + if( mis(part, m->getPCU(), false, true) || 1 == part.net.size() ) { misNumber = iter; part.net.clear(); part.adjPartIds.clear(); } iter++; misSize = (misNumber != -1); - misSize = PCU_Add_Int(misSize); + misSize = m->getPCU()->Add(misSize); } return misNumber; } diff --git a/parma/parma.h b/parma/parma.h index 065477fa7..b9fe005e2 100644 --- a/parma/parma.h +++ b/parma/parma.h @@ -13,6 +13,7 @@ #include "apf.h" #include "apfPartition.h" +#include /** * @brief get entity imbalance @@ -288,6 +289,7 @@ apf::MeshTag* Parma_WeighByMemory(apf::Mesh* m); */ struct Parma_GroupCode { + std::unique_ptr PCUObj; /** * @brief Called withing sub-groups. * @details Within a group, all PCU functions behave @@ -314,7 +316,7 @@ struct Parma_GroupCode * parts are combined into one and then that one is split back * out into (factor) contiguous part ids again. */ -void Parma_ShrinkPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun); +void Parma_ShrinkPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun, pcu::PCU *PCUObj = nullptr); /** * @brief Split the processes into groups of (factor). @@ -327,7 +329,7 @@ void Parma_ShrinkPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun); * apf::remapPartition is used to maintain the mesh structure * during these transitions. */ -void Parma_SplitPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun); +void Parma_SplitPartition(apf::Mesh2* m, int factor, Parma_GroupCode& toRun, pcu::PCU *PCUObj = nullptr); /** * @brief Compute maximal independent set numbering diff --git a/parma/rib/parma_mesh_rib.cc b/parma/rib/parma_mesh_rib.cc index 28b0c5450..31915e86d 100644 --- a/parma/rib/parma_mesh_rib.cc +++ b/parma/rib/parma_mesh_rib.cc @@ -1,4 +1,3 @@ -#include #include "parma_rib.h" #include #include @@ -63,7 +62,7 @@ class RibSplitter : public apf::Splitter virtual apf::Migration* split(apf::MeshTag* weights, double, int multiple) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); int depth; for (depth = 0; (1 << depth) < multiple; ++depth); PCU_ALWAYS_ASSERT((1 << depth) == multiple); @@ -75,8 +74,8 @@ class RibSplitter : public apf::Splitter int p = plan->sending(e); plan->send(e, p + offset); } - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!mesh->getPCU()->Self()) lion_oprint(1,"planned RIB factor %d in %f seconds\n", multiple, t1 - t0); } diff --git a/pcu/CMakeLists.txt b/pcu/CMakeLists.txt index 91627f7ca..51ccbcdea 100644 --- a/pcu/CMakeLists.txt +++ b/pcu/CMakeLists.txt @@ -9,7 +9,7 @@ message(STATUS "PCU_COMPRESS: " ${PCU_COMPRESS}) # Package sources set(SOURCES - pcu.c + pcu_c.cc pcu_aa.c pcu_coll.c pcu_io.c @@ -22,14 +22,16 @@ set(SOURCES pcu_util.c noto/noto_malloc.c reel/reel.c -) + PCU.cc) # Package headers set(HEADERS - PCU.h + PCU_C.h pcu_io.h pcu_util.h reel/reel.h + pcu_defines.h + PCU.h ) # Add the pcu library diff --git a/pcu/PCU.cc b/pcu/PCU.cc new file mode 100644 index 000000000..328628e51 --- /dev/null +++ b/pcu/PCU.cc @@ -0,0 +1,296 @@ +#include "PCU.h" +#include "noto_malloc.h" +#include "pcu_mem.h" +#include "pcu_mpi.h" +#include "pcu_msg.h" +#include "pcu_order.h" +#include "reel.h" +#include +#include /*using POSIX mkdir call for SMB "foo/" path*/ +#include +#include +#include +#include +namespace pcu { + +int PCU::Peers() const noexcept { return pcu_mpi_size(mpi_); } +int PCU::Self() const noexcept { return pcu_mpi_rank(mpi_); } +void PCU::Begin() noexcept { pcu_msg_start(mpi_, msg_); } +int PCU::Pack(int to_rank, const void *data, size_t size) noexcept { + if ((to_rank < 0) || (to_rank >= Peers())) + reel_fail("Invalid rank in Comm_Pack"); + if (size > (size_t)INT_MAX) { + fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds " + "INT_MAX... exiting\n"); + abort(); + } + memcpy(pcu_msg_pack(msg_, to_rank, size), data, size); + return PCU_SUCCESS; +} + +int PCU::Send() noexcept { + pcu_msg_send(mpi_, msg_); + return PCU_SUCCESS; +} +bool PCU::Receive() noexcept { + while (Unpacked()) + if (!Listen()) + return false; + return true; +} +bool PCU::Listen() noexcept { + if (msg_->order) + return pcu_order_receive(mpi_, msg_->order, msg_); + return pcu_msg_receive(mpi_, msg_); +} +int PCU::Sender() noexcept { + if (msg_->order) + return pcu_order_received_from(msg_->order); + return pcu_msg_received_from(msg_); +} +bool PCU::Unpacked() noexcept { + if (msg_->order) + return pcu_order_unpacked(msg_->order); + return pcu_msg_unpacked(msg_); +} +int PCU::Unpack(void *data, size_t size) noexcept { + if (msg_->order) + memcpy(data, pcu_order_unpack(msg_->order, size), size); + else + memcpy(data, pcu_msg_unpack(msg_, size), size); + return PCU_SUCCESS; +} + +int PCU::Write(int to_rank, const void *data, size_t size) noexcept { + if ((to_rank < 0) || (to_rank >= Peers())) + reel_fail("Invalid rank in Comm_Write"); + PCU_MSG_PACK(msg_, to_rank, size); + memcpy(pcu_msg_pack(msg_, to_rank, size), data, size); + return PCU_SUCCESS; +} +bool PCU::Read(int *from_rank, void **data, size_t *size) noexcept { + if (!Receive()) + return false; + *from_rank = Sender(); + Unpack(size, sizeof(*size)); + *data = Extract(*size); + return true; +} +void PCU::Order(bool on) { + if (on && (!msg_->order)) + msg_->order = pcu_order_new(); + if ((!on) && msg_->order) { + pcu_order_free(msg_->order); + msg_->order = NULL; + } +} +void PCU::Barrier() { pcu_barrier(mpi_, &(msg_->coll)); } +int PCU::Or(int c) noexcept { return Max(c); } +int PCU::And(int c) noexcept { return Min(c); } +int PCU::Packed(int to_rank, size_t *size) noexcept { + if ((to_rank < 0) || (to_rank >= Peers())) + reel_fail("Invalid rank in Comm_Packed"); + *size = pcu_msg_packed(msg_, to_rank); + return PCU_SUCCESS; +} +int PCU::From(int *from_rank) noexcept { + if (msg_->order) + *from_rank = pcu_order_received_from(msg_->order); + else + *from_rank = pcu_msg_received_from(msg_); + return PCU_SUCCESS; +} +int PCU::Received(size_t *size) noexcept { + if (msg_->order) + *size = pcu_order_received_size(msg_->order); + else + *size = pcu_msg_received_size(msg_); + return PCU_SUCCESS; +} +void *PCU::Extract(size_t size) noexcept { + if (msg_->order) + return pcu_order_unpack(msg_->order, size); + return pcu_msg_unpack(msg_, size); +} + +static void safe_mkdir(const char *path, mode_t mode) { + int err; + errno = 0; + err = mkdir(path, mode); + if (err != 0 && errno != EEXIST) + reel_fail("PCU: could not create directory \"%s\"\n", path); +} + +static void append(char *s, size_t size, const char *format, ...) { + int len = strlen(s); + va_list ap; + va_start(ap, format); + vsnprintf(s + len, size - len, format, ap); + va_end(ap); +} + +void PCU::DebugOpen() noexcept { + const int fanout = 2048; + const int bufsize = 1024; + char *path = (char *)noto_malloc(bufsize); + path[0] = '\0'; + if (Peers() > fanout) { + mode_t const dir_perm = S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; + strcpy(path, "debug/"); + safe_mkdir(path, dir_perm); + int self = Self(); + append(path, bufsize, "%d/", self / fanout); + if (self % fanout == 0) + safe_mkdir(path, dir_perm); + Barrier(); + } + + append(path, bufsize, "%s", "debug"); + if (!msg_->file) + msg_->file = pcu_open_parallel(GetCHandle(), path, "txt"); + noto_free(path); +} + + + +double GetMem() noexcept { return pcu_get_mem(); } +void Protect() noexcept { reel_protect(); } +double Time() noexcept { return MPI_Wtime(); } + +void PCU::DebugPrint(const char *format, ...) noexcept { + va_list args; + va_start(args, format); + DebugPrint(format, args); + va_end(args); +} +void PCU::DebugPrint(const char *format, va_list args) noexcept { + if (!msg_->file) + return; // Print is a no-op if no file is open + vfprintf(msg_->file, format, args); + fflush(msg_->file); +} +PCU::PCU(MPI_Comm comm) { + mpi_ = new pcu_mpi_t; + msg_ = new pcu_msg; + pcu_mpi_init(comm, mpi_); + pcu_make_msg(msg_); + /* turn ordering on by default, call + PCU_Comm_Order(false) after PCU_Comm_Init + to disable this */ + Order(true); +} +PCU::~PCU() noexcept { + pcu_mpi_finalize(mpi_); + delete mpi_; + pcu_free_msg(msg_); + delete msg_; +} +PCU::PCU(PCU &&other) noexcept { + std::swap(mpi_, other.mpi_); + std::swap(msg_, other.msg_); +} +PCU &PCU::operator=(PCU && other) noexcept { + std::swap(mpi_, other.mpi_); + std::swap(msg_, other.msg_); + return *this; +} +MPI_Comm PCU::GetMPIComm() const noexcept { return mpi_->original_comm; } + +MPI_Comm PCU::SwitchMPIComm(MPI_Comm newcomm) noexcept { + if(newcomm == mpi_->original_comm) { + return mpi_->original_comm; + } + auto original_comm = mpi_->original_comm; + pcu_mpi_finalize(mpi_); + pcu_mpi_init(newcomm, mpi_); + return original_comm; +} + +/* template implementations */ +template void PCU::Add(T *p, size_t n) noexcept { + pcu_allreduce( + mpi_, &(msg_->coll), + [](void *local, void *incoming, size_t size) { + auto *a = static_cast(local); + auto *b = static_cast(incoming); + size_t n = size / sizeof(T); + for (size_t i = 0; i < n; ++i) + a[i] += b[i]; + }, + p, n * sizeof(T)); +} +template T PCU::Add(T p) noexcept { + Add(&p, 1); + return p; +} +template void PCU::Min(T *p, size_t n) noexcept { + pcu_allreduce( + mpi_, &(msg_->coll), + [](void *local, void *incoming, size_t size) { + auto *a = static_cast(local); + auto *b = static_cast(incoming); + size_t n = size / sizeof(T); + for (size_t i = 0; i < n; ++i) + a[i] = std::min(a[i], b[i]); + }, + p, n * sizeof(T)); +} +template T PCU::Min(T p) noexcept { + Min(&p, 1); + return p; +} +template void PCU::Max(T *p, size_t n) noexcept { + pcu_allreduce( + mpi_, &(msg_->coll), + [](void *local, void *incoming, size_t size) { + auto *a = static_cast(local); + auto *b = static_cast(incoming); + size_t n = size / sizeof(T); + for (size_t i = 0; i < n; ++i) + a[i] = std::max(a[i], b[i]); + }, + p, n * sizeof(T)); +} +template T PCU::Max(T p) noexcept { + Max(&p, 1); + return p; +} +template void PCU::Exscan(T *p, size_t n) noexcept { + auto *originals = (T *)noto_malloc(sizeof(T) * n); + for (size_t i = 0; i < n; ++i) + originals[i] = p[i]; + pcu_scan( + mpi_, &(msg_->coll), + [](void *local, void *incoming, size_t size) { + auto *a = static_cast(local); + auto *b = static_cast(incoming); + size_t n = size / sizeof(T); + for (size_t i = 0; i < n; ++i) + a[i] += b[i]; + }, + p, n * sizeof(T)); + // convert inclusive scan to exclusive + for (size_t i = 0; i < n; ++i) + p[i] -= originals[i]; + noto_free(originals); +} +template T PCU::Exscan(T p) noexcept { + Exscan(&p, 1); + return p; +} +#define PCU_EXPL_INST_DECL(T) \ + template void PCU::Add(T * p, size_t n) noexcept; \ + template T PCU::Add(T p) noexcept; \ + template void PCU::Min(T * p, size_t n) noexcept; \ + template T PCU::Min(T p) noexcept; \ + template void PCU::Max(T * p, size_t n) noexcept; \ + template T PCU::Max(T p) noexcept; \ + template void PCU::Exscan(T * p, size_t n) noexcept; \ + template T PCU::Exscan(T p) noexcept; +PCU_EXPL_INST_DECL(int) +PCU_EXPL_INST_DECL(size_t) +PCU_EXPL_INST_DECL(long) +PCU_EXPL_INST_DECL(double) +#undef PCU_EXPL_INST_DECL + +} // namespace pcu diff --git a/pcu/PCU.h b/pcu/PCU.h index f9a53362e..3edf052fc 100644 --- a/pcu/PCU.h +++ b/pcu/PCU.h @@ -1,138 +1,131 @@ -/****************************************************************************** - - Copyright 2011 Scientific Computation Research Center, - Rensselaer Polytechnic Institute. All rights reserved. - - This work is open source software, licensed under the terms of the - BSD license as described in the LICENSE file in the top-level directory. - -*******************************************************************************/ -#ifndef PCU_H -#define PCU_H - -#define PCU_SUCCESS 0 -#define PCU_FAILURE -1 +#ifndef SCOREC_PCU_H +#define SCOREC_PCU_H +#include #include +#include "pcu_defines.h" + +struct pcu_msg_struct; +struct pcu_mpi_struct; + +namespace pcu { +class PCU { +public: + explicit PCU(MPI_Comm comm); + ~PCU() noexcept; + PCU(PCU const &) = delete; + PCU(PCU &&) noexcept; + PCU &operator=(PCU const &) = delete; + PCU &operator=(PCU &&) noexcept; + /** @brief Returns the rank of the current process. + * @return The rank of the current process. + */ + [[nodiscard]] int Self() const noexcept; + /** @brief Returns the number of ranks in the communicator. + * @return The number of ranks in the communicator. + */ + [[nodiscard]] int Peers() const noexcept; + [[nodiscard]] MPI_Comm GetMPIComm() const noexcept; + + [[nodiscard]] PCU_t GetCHandle() {PCU_t h; h.ptr=this; return h;} + /*recommended message passing API*/ + void Begin() noexcept; + int Pack(int to_rank, const void *data, size_t size) noexcept; + template int Pack(int to_rank, T& data) noexcept { + return Pack(to_rank, &(data), sizeof(data)); + } + template int Pack(int to_rank, T*& data) noexcept { + return Pack(to_rank, &(data), sizeof(data)); + } + + int Send() noexcept; + bool Receive() noexcept; + bool Listen() noexcept; + int Sender() noexcept; + bool Unpacked() noexcept; + int Unpack(void *data, size_t size) noexcept; + template int Unpack(T& data) noexcept { + return Unpack(&(data), sizeof(data)); + } + template int Unpack(T*& data) noexcept { + return Unpack(&(data), sizeof(data)); + } + /*IPComMan replacement API*/ + int Write(int to_rank, const void *data, size_t size) noexcept; + bool Read(int *from_rank, void **data, size_t *size) noexcept; + + /*turns deterministic ordering for the + above API on/off*/ + void Order(bool on); + + /*collective operations*/ + void Barrier(); + template void Add(T *p, size_t n) noexcept; + template [[nodiscard]] T Add(T p) noexcept; + template void Min(T *p, size_t n) noexcept; + template [[nodiscard]] T Min(T p) noexcept; + template void Max(T *p, size_t n) noexcept; + template [[nodiscard]] T Max(T p) noexcept; + template void Exscan(T *p, size_t n) noexcept; + template [[nodiscard]] T Exscan(T p) noexcept; + + /*bitwise operations*/ + [[nodiscard]] int Or(int c) noexcept; + [[nodiscard]] int And(int c) noexcept; + + /*lesser-used APIs*/ + int Packed(int to_rank, size_t *size) noexcept; + int From(int *from_rank) noexcept; + int Received(size_t *size) noexcept; + void *Extract(size_t size) noexcept; + + void DebugPrint(const char* format, ...) noexcept PCU_FORMAT_ATTRIBUTE(2, 3) + void DebugPrint(const char* format, va_list args) noexcept; + /* Debug functions */ + void DebugOpen() noexcept; + + MPI_Comm SwitchMPIComm(MPI_Comm) noexcept; + + //struct MPIComms { + // MPI_Comm original; + // MPI_Comm user; + // MPI_Comm coll; + //}; + // takes ownership of newcomms.user & newcomms.coll + // user responsibility to free returned user/coll comm + //MPIComms SwitchMPIComms(MPIComms& newcomms) noexcept; + +private: + pcu_msg_struct *msg_; + pcu_mpi_struct *mpi_; +}; +/*stack trace helpers using GNU/Linux*/ +void Protect() noexcept; +/*Memory usage*/ +[[nodiscard]] double GetMem() noexcept; +/*MPI_Wtime() equivalent*/ +[[nodiscard]] double Time() noexcept; -#ifdef __cplusplus -#include -#include -extern "C" { -#else -#include -#include -#include -#endif - -/*library init/finalize*/ -int PCU_Comm_Init(void); -int PCU_Comm_Free(void); - -/*rank/size functions*/ -int PCU_Comm_Self(void); -int PCU_Comm_Peers(void); - -/*recommended message passing API*/ -void PCU_Comm_Begin(void); -int PCU_Comm_Pack(int to_rank, const void* data, size_t size); -#define PCU_COMM_PACK(to_rank,object)\ -PCU_Comm_Pack(to_rank,&(object),sizeof(object)) -int PCU_Comm_Send(void); -bool PCU_Comm_Receive(void); -bool PCU_Comm_Listen(void); -int PCU_Comm_Sender(void); -bool PCU_Comm_Unpacked(void); -int PCU_Comm_Unpack(void* data, size_t size); -#define PCU_COMM_UNPACK(object)\ -PCU_Comm_Unpack(&(object),sizeof(object)) - -/*turns deterministic ordering for the - above API on/off*/ -void PCU_Comm_Order(bool on); - -/*collective operations*/ -void PCU_Barrier(void); -void PCU_Add_Doubles(double* p, size_t n); -double PCU_Add_Double(double x); -void PCU_Min_Doubles(double* p, size_t n); -double PCU_Min_Double(double x); -void PCU_Max_Doubles(double* p, size_t n); -double PCU_Max_Double(double x); -void PCU_Add_Ints(int* p, size_t n); -int PCU_Add_Int(int x); -void PCU_Add_Longs(long* p, size_t n); -long PCU_Add_Long(long x); -void PCU_Exscan_Ints(int* p, size_t n); -int PCU_Exscan_Int(int x); -void PCU_Exscan_Longs(long* p, size_t n); -long PCU_Exscan_Long(long x); -void PCU_Add_SizeTs(size_t* p, size_t n); -size_t PCU_Add_SizeT(size_t x); -void PCU_Min_SizeTs(size_t* p, size_t n); -size_t PCU_Min_SizeT(size_t x); -void PCU_Max_SizeTs(size_t* p, size_t n); -size_t PCU_Max_SizeT(size_t x); -void PCU_Min_Ints(int* p, size_t n); -int PCU_Min_Int(int x); -void PCU_Max_Ints(int* p, size_t n); -int PCU_Max_Int(int x); -void PCU_Max_Longs(long* p, size_t n); -long PCU_Max_Long(long x); -int PCU_Or(int c); -int PCU_And(int c); - -/*process-level self/peers (mpi wrappers)*/ -int PCU_Proc_Self(void); -int PCU_Proc_Peers(void); - -/*IPComMan replacement API*/ -int PCU_Comm_Write(int to_rank, const void* data, size_t size); -#define PCU_COMM_WRITE(to,data) \ -PCU_Comm_Write(to,&(data),sizeof(data)) -bool PCU_Comm_Read(int* from_rank, void** data, size_t* size); - -/*Debug file I/O API*/ -void PCU_Debug_Open(void); -#ifdef __GNUC__ -void PCU_Debug_Print(const char* format, ...) - __attribute__((format(printf,1,2))); -#else -void PCU_Debug_Print(const char* format, ...); -#endif - -/*lesser-used APIs*/ -bool PCU_Comm_Initialized(void); -int PCU_Comm_Packed(int to_rank, size_t* size); -int PCU_Comm_From(int* from_rank); -int PCU_Comm_Received(size_t* size); -void* PCU_Comm_Extract(size_t size); -int PCU_Comm_Rank(int* rank); -int PCU_Comm_Size(int* size); - -/*deprecated method enum*/ -#ifdef __cplusplus -enum PCU_Method { PCU_GLOBAL_METHOD, PCU_LOCAL_METHOD }; -#else -typedef enum { PCU_GLOBAL_METHOD, PCU_LOCAL_METHOD } PCU_Method; -#endif -int PCU_Comm_Start(PCU_Method method); - -/*special MPI_Comm replacement API*/ -void PCU_Switch_Comm(MPI_Comm new_comm); -MPI_Comm PCU_Get_Comm(void); +PCU* PCU_GetGlobal(); -/*stack trace helpers using GNU/Linux*/ -void PCU_Protect(void); +/* explicit instantiations of template functions */ +#define PCU_EXPL_INST_DECL(T) \ + extern template void PCU::Add(T * p, size_t n) noexcept; \ + extern template T PCU::Add(T p) noexcept; \ + extern template void PCU::Min(T * p, size_t n) noexcept; \ + extern template T PCU::Min(T p) noexcept; \ + extern template void PCU::Max(T * p, size_t n) noexcept; \ + extern template T PCU::Max(T p) noexcept; \ + extern template void PCU::Exscan(T * p, size_t n) noexcept; \ + extern template T PCU::Exscan(T p) noexcept; +PCU_EXPL_INST_DECL(int) +PCU_EXPL_INST_DECL(size_t) +PCU_EXPL_INST_DECL(long) +PCU_EXPL_INST_DECL(double) +#undef PCU_EXPL_INST_DECL -/*MPI_Wtime() equivalent*/ -double PCU_Time(void); +} // namespace pcu -/*Memory usage*/ -double PCU_GetMem(void); +#endif // PCUOBJ_H -#ifdef __cplusplus -} /* extern "C" */ -#endif -#endif diff --git a/pcu/PCU_C.h b/pcu/PCU_C.h new file mode 100644 index 000000000..e8a05169c --- /dev/null +++ b/pcu/PCU_C.h @@ -0,0 +1,120 @@ +#ifndef PCU_C_H +#define PCU_C_H +#include "pcu_defines.h" +#include + +#ifdef __cplusplus +#include +#include +extern "C" { +#else +#include +#include +#include +#endif + +typedef struct PCU_t PCU_t; + +int PCU_Comm_Init(PCU_t* h); +int PCU_Comm_Free(PCU_t* h); + +/*rank/size functions*/ +int PCU_Comm_Self(PCU_t h); +int PCU_Comm_Peers(PCU_t h); + +/*recommended message passing API*/ +void PCU_Comm_Begin(PCU_t h); +int PCU_Comm_Pack(PCU_t h, int to_rank, const void* data, size_t size); +#define PCU_COMM_PACK(handle, to_rank,object)\ +PCU_Comm_Pack(handle, to_rank,&(object),sizeof(object)) +int PCU_Comm_Send(PCU_t h); +bool PCU_Comm_Receive(PCU_t h); +bool PCU_Comm_Listen(PCU_t h); +int PCU_Comm_Sender(PCU_t h); +bool PCU_Comm_Unpacked(PCU_t h); +int PCU_Comm_Unpack(PCU_t h, void* data, size_t size); +#define PCU_COMM_UNPACK(handle, object)\ +PCU_Comm_Unpack(handle, &(object),sizeof(object)) + +/*turns deterministic ordering for the + above API on/off*/ +void PCU_Comm_Order(PCU_t h, bool on); + +/*collective operations*/ +void PCU_Barrier(PCU_t h); +void PCU_Add_Doubles(PCU_t h, double* p, size_t n); +double PCU_Add_Double(PCU_t h, double x); +void PCU_Min_Doubles(PCU_t h, double* p, size_t n); +double PCU_Min_Double(PCU_t h, double x); +void PCU_Max_Doubles(PCU_t h, double* p, size_t n); +double PCU_Max_Double(PCU_t h, double x); +void PCU_Add_Ints(PCU_t h, int* p, size_t n); +int PCU_Add_Int(PCU_t h, int x); +void PCU_Add_Longs(PCU_t h, long* p, size_t n); +long PCU_Add_Long(PCU_t h, long x); +void PCU_Exscan_Ints(PCU_t h, int* p, size_t n); +int PCU_Exscan_Int(PCU_t h, int x); +void PCU_Exscan_Longs(PCU_t h, long* p, size_t n); +long PCU_Exscan_Long(PCU_t h, long x); +void PCU_Add_SizeTs(PCU_t h, size_t* p, size_t n); +size_t PCU_Add_SizeT(PCU_t h, size_t x); +void PCU_Min_SizeTs(PCU_t h, size_t* p, size_t n); +size_t PCU_Min_SizeT(PCU_t h, size_t x); +void PCU_Max_SizeTs(PCU_t h, size_t* p, size_t n); +size_t PCU_Max_SizeT(PCU_t h, size_t x); +void PCU_Min_Ints(PCU_t h, int* p, size_t n); +int PCU_Min_Int(PCU_t h, int x); +void PCU_Max_Ints(PCU_t h, int* p, size_t n); +int PCU_Max_Int(PCU_t h, int x); +void PCU_Max_Longs(PCU_t h, long* p, size_t n); +long PCU_Max_Long(PCU_t h, long x); +int PCU_Or(PCU_t h, int c); +int PCU_And(PCU_t h, int c); + +/*process-level self/peers (mpi wrappers)*/ +int PCU_Proc_Self(PCU_t h); +int PCU_Proc_Peers(PCU_t h); + +/*IPComMan replacement API*/ +int PCU_Comm_Write(PCU_t h, int to_rank, const void* data, size_t size); +#define PCU_COMM_WRITE(handle,to,data) \ +PCU_Comm_Write(handle, to,&(data),sizeof(data)) +bool PCU_Comm_Read(PCU_t h, int* from_rank, void** data, size_t* size); + +/*Debug file I/O API*/ +void PCU_Debug_Open(PCU_t h); + +void PCU_Debug_Print(PCU_t h, const char* format, ...) PCU_FORMAT_ATTRIBUTE(2,3); +/*lesser-used APIs*/ +bool PCU_Comm_Initialized(PCU_t h); +int PCU_Comm_Packed(PCU_t h ,int to_rank, size_t* size); +int PCU_Comm_From(PCU_t h, int* from_rank); +int PCU_Comm_Received(PCU_t h, size_t* size); +void* PCU_Comm_Extract(PCU_t h, size_t size); +int PCU_Comm_Rank(PCU_t h, int* rank); +int PCU_Comm_Size(PCU_t h, int* size); + +/*deprecated method enum*/ + +/*special MPI_Comm replacement API*/ +void PCU_Switch_Comm(PCU_t h, MPI_Comm new_comm); +MPI_Comm PCU_Get_Comm(PCU_t h); + +/*stack trace helpers using GNU/Linux*/ +void PCU_Protect(void); + +/*MPI_Wtime() equivalent*/ +double PCU_Time(void); + +/*Memory usage*/ +double PCU_GetMem(void); + +/*Access global variable*/ +PCU_t PCU_Get_Global_Handle(void); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif \ No newline at end of file diff --git a/pcu/noto/noto_malloc.h b/pcu/noto/noto_malloc.h index e1f60ed36..42a0cd24a 100644 --- a/pcu/noto/noto_malloc.h +++ b/pcu/noto/noto_malloc.h @@ -1,8 +1,8 @@ -/****************************************************************************** +/****************************************************************************** - Copyright 2015 Scientific Computation Research Center, + Copyright 2015 Scientific Computation Research Center, Rensselaer Polytechnic Institute. All rights reserved. - + This work is open source software, licensed under the terms of the BSD license as described in the LICENSE file in the top-level directory. @@ -10,6 +10,10 @@ #ifndef NOTO_MALLOC_H #define NOTO_MALLOC_H +#ifdef __cplusplus +extern "C" { +#endif + #include void* noto_malloc(size_t size); @@ -17,6 +21,7 @@ void* noto_malloc(size_t size); void* noto_realloc(void* p, size_t size); void noto_free(void* p); size_t noto_malloced(void); - +#ifdef __cplusplus +} +#endif #endif - diff --git a/pcu/pcu.c b/pcu/pcu.c deleted file mode 100644 index b6c30f901..000000000 --- a/pcu/pcu.c +++ /dev/null @@ -1,775 +0,0 @@ -/****************************************************************************** - - Copyright 2011 Scientific Computation Research Center, - Rensselaer Polytechnic Institute. All rights reserved. - - This work is open source software, licensed under the terms of the - BSD license as described in the LICENSE file in the top-level directory. - -*******************************************************************************/ -/** \file pcu.c - \brief The PCU communication interface */ -/** \page pcu PCU - PCU (the Parallel Control Utility) is a library for parallel computation - based on MPI. - PCU provides three things to users: - 1. A hybrid phased message passing system - 2. Hybrid collective operations - - Phased message passing is similar to Bulk Synchronous Parallel. - All messages are exchanged in a phase, which is a collective operation - involving all threads in the parallel program. - During a phase, the following events happen in sequence: - 1. All threads send non-blocking messages to other threads - 2. All threads receive all messages sent to them during this phase - PCU provides termination detection, which is the ability to detect when all - messages have been received without prior knowledge of which threads - are sending to which. - - The API documentation is here: pcu.c -*/ - -#include -#include -#include "PCU.h" -#include "pcu_msg.h" -#include "pcu_pmpi.h" -#include "pcu_order.h" -#include "noto_malloc.h" -#include "reel.h" -#include /*required for mode_t for mkdir on some systems*/ -#include /*using POSIX mkdir call for SMB "foo/" path*/ -#include /* for checking the error from mkdir */ -#include /*INT_MAX*/ -#include /*abort*/ - -enum state { uninit, init }; -static enum state global_state = uninit; -static pcu_msg global_pmsg; - -static pcu_msg* get_msg() -{ - return &global_pmsg; -} - -/** \brief Initializes the PCU library. - \details This function must be called by all MPI processes before - calling any other PCU functions. - MPI_Init or MPI_Init_thread should be called before this function. - */ -int PCU_Comm_Init(void) -{ - if (global_state != uninit) - reel_fail("nested calls to Comm_Init"); - pcu_pmpi_init(MPI_COMM_WORLD); - pcu_set_mpi(&pcu_pmpi); - pcu_make_msg(&global_pmsg); - global_state = init; - /* turn ordering on by default, call - PCU_Comm_Order(false) after PCU_Comm_Init - to disable this */ - PCU_Comm_Order(true); - return PCU_SUCCESS; -} - -/** \brief Frees all PCU library structures. - \details This function must be called by all MPI processes after all - other calls to PCU, and before calling MPI_Finalize. - */ -int PCU_Comm_Free(void) -{ - if (global_state == uninit) - reel_fail("Comm_Free called before Comm_Init"); - if (global_pmsg.order) - pcu_order_free(global_pmsg.order); - pcu_free_msg(&global_pmsg); - pcu_pmpi_finalize(); - global_state = uninit; - return PCU_SUCCESS; -} - -/** \brief Returns the communication rank of the calling thread. - \details when called from a non-threaded MPI process, this function is - equivalent to MPI_Comm_rank(MPI_COMM_WORLD,rank). - - Ranks are consecutive from 0 to \f$pt-1\f$ for a program with - \f$p\f$ processes and \f$t\f$ threads per process. - Ranks are contiguous within a process, so that the \f$t\f$ threads in process - \f$i\f$ are numbered from \f$ti\f$ to \f$ti+t-1\f$. - */ -int PCU_Comm_Self(void) -{ - if (global_state == uninit) - reel_fail("Comm_Self called before Comm_Init"); - return pcu_mpi_rank(); -} - -/** \brief Returns the number of threads in the program. - \details when called from a non-threaded MPI process, this function is - equivalent to MPI_Comm_size(MPI_COMM_WORLD,size). - */ -int PCU_Comm_Peers(void) -{ - if (global_state == uninit) - reel_fail("Comm_Peers called before Comm_Init"); - return pcu_mpi_size(); -} - -/** \brief Begins a PCU communication phase. - \details This function must be called by all threads in the MPI job - at the beginning of each phase of communication. - After calling this function, each thread may call functions like - PCU_Comm_Pack or PCU_Comm_Write. -*/ -void PCU_Comm_Begin(void) -{ - if (global_state == uninit) - reel_fail("Comm_Begin called before Comm_Init"); - pcu_msg_start(get_msg()); -} - -/** \brief Packs data to be sent to \a to_rank. - \details This function appends the block of \a size bytes starting - at \a data to the buffer being sent to \a to_rank. - This function should be called after PCU_Comm_Start and before - PCU_Comm_Send. - */ -int PCU_Comm_Pack(int to_rank, const void* data, size_t size) -{ - if (global_state == uninit) - reel_fail("Comm_Pack called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size())) - reel_fail("Invalid rank in Comm_Pack"); - if ( size > (size_t)INT_MAX ) { - fprintf(stderr, "ERROR Attempting to pack a PCU message whose size exceeds INT_MAX... exiting\n"); - abort(); - } - memcpy(pcu_msg_pack(get_msg(),to_rank,size),data,size); - return PCU_SUCCESS; -} - -/** \brief Sends all buffers for this communication phase. - \details This function should be called by all threads in the MPI job - after calls to PCU_Comm_Pack or PCU_Comm_Write and before calls - to PCU_Comm_Listen or PCU_Comm_Read. - All buffers from this thread are sent out and receiving - may begin after this call. - */ -int PCU_Comm_Send(void) -{ - if (global_state == uninit) - reel_fail("Comm_Send called before Comm_Init"); - pcu_msg_send(get_msg()); - return PCU_SUCCESS; -} - -/** \brief Tries to receive a buffer for this communication phase. - \details Either this function or PCU_Comm_Read should be called at least - once by all threads during the communication phase, after PCU_Comm_Send - is called. The result will be false if and only if the communication phase - is over and there are no more buffers to receive. - Otherwise, a buffer was received. - Its contents are retrievable through PCU_Comm_Unpack, and its metadata through - PCU_Comm_Sender and PCU_Comm_Received. - Users should unpack all data from this buffer before calling this function - again, because the previously received buffer is destroyed by the call. - */ -bool PCU_Comm_Listen(void) -{ - if (global_state == uninit) - reel_fail("Comm_Listen called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - return pcu_order_receive(m->order, m); - return pcu_msg_receive(m); -} - -/** \brief Returns in * \a from_rank the sender of the current received buffer. - \details This function should be called after a successful PCU_Comm_Listen. - */ -int PCU_Comm_Sender(void) -{ - if (global_state == uninit) - reel_fail("Comm_Sender called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - return pcu_order_received_from(m->order); - return pcu_msg_received_from(m); -} - -/** \brief Returns true if the current received buffer has been unpacked. - \details This function should be called after a successful PCU_Comm_Listen. - */ -bool PCU_Comm_Unpacked(void) -{ - if (global_state == uninit) - reel_fail("Comm_Unpacked called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - return pcu_order_unpacked(m->order); - return pcu_msg_unpacked(m); -} - -/** \brief Unpacks a block of data from the current received buffer. - \details This function should be called after a successful PCU_Comm_Listen. - \a data must point to a block of memory of at least \a size bytes, into - which the next \a size bytes of the current received buffer will be written. - Subsequent calls will begin unpacking where this call left off, - so that the entire received buffer can be unpacked by a sequence of calls to - this function. - Users must ensure that there remain \a size bytes to be unpacked, - PCU_Comm_Unpacked can help with this. - */ -int PCU_Comm_Unpack(void* data, size_t size) -{ - if (global_state == uninit) - reel_fail("Comm_Unpack called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - memcpy(data,pcu_order_unpack(m->order,size),size); - else - memcpy(data,pcu_msg_unpack(m,size),size); - return PCU_SUCCESS; -} - -void PCU_Comm_Order(bool on) -{ - if (global_state == uninit) - reel_fail("Comm_Order called before Comm_Init"); - pcu_msg* m = get_msg(); - if (on && (!m->order)) - m->order = pcu_order_new(); - if ((!on) && m->order) { - pcu_order_free(m->order); - m->order = NULL; - } -} - -/** \brief Blocking barrier over all threads. */ -void PCU_Barrier(void) -{ - if (global_state == uninit) - reel_fail("Barrier called before Comm_Init"); - pcu_barrier(&(get_msg()->coll)); -} - -/** \brief Performs an Allreduce sum of double arrays. - \details This function must be called by all ranks at - the same time. \a p must point to an array of \a n doubles. - After this call, p[i] will contain the sum of all p[i]'s - given by each rank. - */ -void PCU_Add_Doubles(double* p, size_t n) -{ - if (global_state == uninit) - reel_fail("Add_Doubles called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_add_doubles,p,n*sizeof(double)); -} - -double PCU_Add_Double(double x) -{ - double a[1]; - a[0] = x; - PCU_Add_Doubles(a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce minimum of double arrays. - */ -void PCU_Min_Doubles(double* p, size_t n) -{ - if (global_state == uninit) - reel_fail("Min_Doubles called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_min_doubles,p,n*sizeof(double)); -} - -double PCU_Min_Double(double x) -{ - double a[1]; - a[0] = x; - PCU_Min_Doubles(a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce maximum of double arrays. - */ -void PCU_Max_Doubles(double* p, size_t n) -{ - if (global_state == uninit) - reel_fail("Max_Doubles called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_max_doubles,p,n*sizeof(double)); -} - -double PCU_Max_Double(double x) -{ - double a[1]; - a[0] = x; - PCU_Max_Doubles(a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce sum of integers - */ -void PCU_Add_Ints(int* p, size_t n) -{ - if (global_state == uninit) - reel_fail("Add_Ints called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); -} - -int PCU_Add_Int(int x) -{ - int a[1]; - a[0] = x; - PCU_Add_Ints(a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce sum of long integers - */ -void PCU_Add_Longs(long* p, size_t n) -{ - if (global_state == uninit) - reel_fail("Add_Longs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); -} - -long PCU_Add_Long(long x) -{ - long a[1]; - a[0] = x; - PCU_Add_Longs(a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce sum of size_t unsigned integers - */ -void PCU_Add_SizeTs(size_t* p, size_t n) -{ - if (global_state == uninit) - reel_fail("Add_SizeTs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_add_sizets,p,n*sizeof(size_t)); -} - -size_t PCU_Add_SizeT(size_t x) -{ - size_t a[1]; - a[0] = x; - PCU_Add_SizeTs(a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce minimum of size_t unsigned integers - */ -void PCU_Min_SizeTs(size_t* p, size_t n) { - if (global_state == uninit) - reel_fail("Min_SizeTs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_min_sizets,p,n*sizeof(size_t)); -} - -size_t PCU_Min_SizeT(size_t x) { - size_t a[1]; - a[0] = x; - PCU_Min_SizeTs(a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce maximum of size_t unsigned integers - */ -void PCU_Max_SizeTs(size_t* p, size_t n) { - if (global_state == uninit) - reel_fail("Max_SizeTs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_max_sizets,p,n*sizeof(size_t)); -} - -size_t PCU_Max_SizeT(size_t x) { - size_t a[1]; - a[0] = x; - PCU_Max_SizeTs(a, 1); - return a[0]; -} - -/** \brief Performs an exclusive prefix sum of integer arrays. - \details This function must be called by all ranks at - the same time. \a p must point to an array of \a n integers. - After this call, p[i] will contain the sum of all p[i]'s - given by ranks lower than the calling rank. - */ -void PCU_Exscan_Ints(int* p, size_t n) -{ - if (global_state == uninit) - reel_fail("Exscan_Ints called before Comm_Init"); - int* originals; - NOTO_MALLOC(originals,n); - for (size_t i=0; i < n; ++i) - originals[i] = p[i]; - pcu_scan(&(get_msg()->coll),pcu_add_ints,p,n*sizeof(int)); - //convert inclusive scan to exclusive - for (size_t i=0; i < n; ++i) - p[i] -= originals[i]; - noto_free(originals); -} - -int PCU_Exscan_Int(int x) -{ - int a[1]; - a[0] = x; - PCU_Exscan_Ints(a, 1); - return a[0]; -} - -/** \brief See PCU_Exscan_Ints */ -void PCU_Exscan_Longs(long* p, size_t n) -{ - if (global_state == uninit) - reel_fail("Exscan_Longs called before Comm_Init"); - long* originals; - NOTO_MALLOC(originals,n); - for (size_t i=0; i < n; ++i) - originals[i] = p[i]; - pcu_scan(&(get_msg()->coll),pcu_add_longs,p,n*sizeof(long)); - //convert inclusive scan to exclusive - for (size_t i=0; i < n; ++i) - p[i] -= originals[i]; - noto_free(originals); -} - -long PCU_Exscan_Long(long x) -{ - long a[1]; - a[0] = x; - PCU_Exscan_Longs(a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce minimum of int arrays. - */ -void PCU_Min_Ints(int* p, size_t n) -{ - if (global_state == uninit) - reel_fail("Min_Ints called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_min_ints,p,n*sizeof(int)); -} - -int PCU_Min_Int(int x) -{ - int a[1]; - a[0] = x; - PCU_Min_Ints(a, 1); - return a[0]; -} - -/** \brief Performs an Allreduce maximum of int arrays. - */ -void PCU_Max_Ints(int* p, size_t n) -{ - if (global_state == uninit) - reel_fail("Max_Ints called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_max_ints,p,n*sizeof(int)); -} - -int PCU_Max_Int(int x) -{ - int a[1]; - a[0] = x; - PCU_Max_Ints(a, 1); - return a[0]; -} -/** \brief Performs an Allreduce maximum of long arrays. - */ -void PCU_Max_Longs(long* p, size_t n) -{ - if (global_state == uninit) - reel_fail("Max_Longs called before Comm_Init"); - pcu_allreduce(&(get_msg()->coll),pcu_max_longs,p,n*sizeof(long)); -} - -long PCU_Max_Long(long x) -{ - long a[1]; - a[0] = x; - PCU_Max_Longs(a, 1); - return a[0]; -} - -/** \brief Performs a parallel logical OR reduction - */ -int PCU_Or(int c) -{ - return PCU_Max_Int(c); -} - -/** \brief Performs a parallel logical AND reduction - */ -int PCU_And(int c) -{ - return PCU_Min_Int(c); -} - -/** \brief Returns the unique rank of the calling process. - */ -int PCU_Proc_Self(void) -{ - if (global_state == uninit) - reel_fail("Proc_Self called before Comm_Init"); - return pcu_pmpi_rank(); -} - -/** \brief Returns the number of processes. - */ -int PCU_Proc_Peers(void) -{ - if (global_state == uninit) - reel_fail("Proc_Peers called before Comm_Init"); - return pcu_pmpi_size(); -} - -/** \brief Similar to PCU_Comm_Self, returns the rank as an argument. - */ -int PCU_Comm_Rank(int* rank) -{ - if (global_state == uninit) - reel_fail("Comm_Rank called before Comm_Init"); - *rank = pcu_mpi_rank(); - return PCU_SUCCESS; -} - -/** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ -int PCU_Comm_Size(int* size) -{ - if (global_state == uninit) - reel_fail("Comm_Size called before Comm_Init"); - *size = pcu_mpi_size(); - return PCU_SUCCESS; -} - -/** \brief Returns true iff PCU has been initialized */ -bool PCU_Comm_Initialized(void) -{ - return global_state == init; -} - -/** \brief Deprecated, see PCU_Comm_Begin. - */ -int PCU_Comm_Start(PCU_Method method) -{ - (void)method; //warning silencer - if (global_state == uninit) - reel_fail("Comm_Start called before Comm_Init"); - pcu_msg_start(get_msg()); - return PCU_SUCCESS; -} - -/** \brief Returns in * \a size the number of bytes being sent to \a to_rank. - \details Returns the size of the buffer being sent to \a to_rank. - This function should be called after PCU_Comm_Start and before - PCU_Comm_Send. - */ -int PCU_Comm_Packed(int to_rank, size_t* size) -{ - if (global_state == uninit) - reel_fail("Comm_Packed called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size())) - reel_fail("Invalid rank in Comm_Packed"); - *size = pcu_msg_packed(get_msg(),to_rank); - return PCU_SUCCESS; -} - -/** \brief Packs a message to be sent to \a to_rank. - \details This function packs a message into the buffer being sent - to \a to_rank. - Messages packed by this function can be received using the function - PCU_Comm_Read. - This function should be called after PCU_Comm_Start and before - PCU_Comm_Send. - If this function is used, PCU_Comm_Pack should not be used. - */ -int PCU_Comm_Write(int to_rank, const void* data, size_t size) -{ - if (global_state == uninit) - reel_fail("Comm_Write called before Comm_Init"); - if ((to_rank < 0)||(to_rank >= pcu_mpi_size())) - reel_fail("Invalid rank in Comm_Write"); - pcu_msg* msg = get_msg(); - PCU_MSG_PACK(msg,to_rank,size); - memcpy(pcu_msg_pack(msg,to_rank,size),data,size); - return PCU_SUCCESS; -} - -/** \brief Convenience wrapper over Listen and Unpacked */ -bool PCU_Comm_Receive(void) -{ - while (PCU_Comm_Unpacked()) - if (!PCU_Comm_Listen()) - return false; - return true; -} - -/** \brief Receives a message for this communication phase. - \details This function tries to receive a message packed by - PCU_Comm_Write. - If a the communication phase is over and there are no more - messages to receive, this function returns false. - Otherwise, * \a from_rank will be the rank which sent the message, - * \a data will point to the start of the message data, and - * \a size will be the number of bytes of message data. - If this function is used, PCU_Comm_Receive should not be used. - Note that the address * \a data points into a PCU buffer, so - it is strongly recommended that this data be read and not modified. - */ -bool PCU_Comm_Read(int* from_rank, void** data, size_t* size) -{ - if (global_state == uninit) - reel_fail("Comm_Read called before Comm_Init"); - if (!PCU_Comm_Receive()) - return false; - *from_rank = PCU_Comm_Sender(); - PCU_COMM_UNPACK(*size); - *data = PCU_Comm_Extract(*size); - return true; -} - -static void safe_mkdir(const char* path, mode_t mode) -{ - int err; - errno = 0; - err = mkdir(path, mode); - if (err != 0 && errno != EEXIST) - reel_fail("PCU: could not create directory \"%s\"\n", path); -} - -static void append(char* s, size_t size, const char* format, ...) -{ - int len = strlen(s); - va_list ap; - va_start(ap, format); - vsnprintf(s + len, size - len, format, ap); - va_end(ap); -} - - -void PCU_Debug_Open(void) -{ - if (global_state == uninit) - reel_fail("Debug_Open called before Comm_Init"); - - const int fanout = 2048; - const int bufsize = 1024; - char* path = noto_malloc(bufsize); - path[0] = '\0'; - if (PCU_Comm_Peers() > fanout) { - mode_t const dir_perm = S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH; - strcpy(path, "debug/"); - safe_mkdir(path, dir_perm); - int self = PCU_Comm_Self(); - append(path, bufsize, "%d/", self / fanout); - if (self % fanout == 0) - safe_mkdir(path, dir_perm); - PCU_Barrier(); - } - - append(path,bufsize, "%s", "debug"); - pcu_msg* msg = get_msg(); - if ( ! msg->file) - msg->file = pcu_open_parallel(path,"txt"); - noto_free(path); -} - -/** \brief like fprintf, contents go to debugN.txt */ -void PCU_Debug_Print(const char* format, ...) -{ - if (global_state == uninit) - reel_fail("Debug_Print called before Comm_Init"); - pcu_msg* msg = get_msg(); - if ( ! msg->file) - return; //Print is a no-op if no file is open - va_list ap; - va_start(ap,format); - vfprintf(msg->file,format,ap); - va_end(ap); - fflush(msg->file); -} - -/** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ -int PCU_Comm_From(int* from_rank) -{ - if (global_state == uninit) - reel_fail("Comm_From called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - *from_rank = pcu_order_received_from(m->order); - else - *from_rank = pcu_msg_received_from(m); - return PCU_SUCCESS; -} - -/** \brief Returns in * \a size the bytes in the current received buffer - \details This function should be called after a successful PCU_Comm_Receive. - The size returned will be the total received size regardless of how - much unpacking has been done. - */ -int PCU_Comm_Received(size_t* size) -{ - if (global_state == uninit) - reel_fail("Comm_Received called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - *size = pcu_order_received_size(m->order); - else - *size = pcu_msg_received_size(m); - return PCU_SUCCESS; -} - -/** \brief Extracts a block of data from the current received buffer. - \details This function should be called after a successful PCU_Comm_Receive. - The next \a size bytes of the current received buffer are unpacked, - and an internal pointer to that data is returned. - The returned pointer must not be freed by the user. - */ -void* PCU_Comm_Extract(size_t size) -{ - if (global_state == uninit) - reel_fail("Comm_Extract called before Comm_Init"); - pcu_msg* m = get_msg(); - if (m->order) - return pcu_order_unpack(m->order,size); - return pcu_msg_unpack(m,size); -} - -/** \brief Reinitializes PCU with a new MPI communicator. - \details All of PCU's logic is based off two duplicates - of this communicator, so you can safely get PCU to act - on sub-groups of processes using this function. - This call should be collective over all processes - in the previous communicator. - */ -void PCU_Switch_Comm(MPI_Comm new_comm) -{ - if (global_state == uninit) - reel_fail("Switch_Comm called before Comm_Init"); - pcu_pmpi_switch(new_comm); -} - -/** \brief Return the current MPI communicator - \details Returns the communicator given to the - most recent PCU_Switch_Comm call, or MPI_COMM_WORLD - otherwise. - */ -MPI_Comm PCU_Get_Comm(void) -{ - if (global_state == uninit) - reel_fail("Get_Comm called before Comm_Init"); - return pcu_pmpi_comm(); -} - -/** \brief Return the time in seconds since some time in the past - */ -double PCU_Time(void) -{ - return MPI_Wtime(); -} - -void PCU_Protect(void) -{ - reel_protect(); -} diff --git a/pcu/pcu_buffer.h b/pcu/pcu_buffer.h index e8ff30d22..7695eacee 100644 --- a/pcu/pcu_buffer.h +++ b/pcu/pcu_buffer.h @@ -12,6 +12,9 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif typedef struct { @@ -28,5 +31,8 @@ void* pcu_walk_buffer(pcu_buffer* b, size_t size); bool pcu_buffer_walked(pcu_buffer* b); void pcu_resize_buffer(pcu_buffer* b, size_t size); void pcu_set_buffer(pcu_buffer* b, void* p, size_t size); +#ifdef __cplusplus +} +#endif #endif diff --git a/pcu/pcu_byteorder.h b/pcu/pcu_byteorder.h index 6621b01cb..7ca91c9dd 100644 --- a/pcu/pcu_byteorder.h +++ b/pcu/pcu_byteorder.h @@ -6,6 +6,10 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif + #if CHAR_BIT != 8 #error "unsupported char size" #endif @@ -22,4 +26,8 @@ static const union { #define PCU_HOST_ORDER (pcu_host_order.value) +#ifdef __cplusplus +} +#endif + #endif diff --git a/pcu/pcu_c.cc b/pcu/pcu_c.cc new file mode 100644 index 000000000..c763cb19f --- /dev/null +++ b/pcu/pcu_c.cc @@ -0,0 +1,503 @@ +#include "PCU_C.h" +#include "PCU.h" +#include "reel.h" +#include + + + +extern "C" { + +int PCU_Comm_Init(PCU_t* h) { + if (h->ptr != nullptr) + reel_fail("nested calls to Comm_Init"); + pcu::PCU* pcu_object = new pcu::PCU(MPI_COMM_WORLD); + h->ptr = static_cast(pcu_object); + return PCU_SUCCESS; +} + +int PCU_Comm_Free(PCU_t* h) { + if (h->ptr == nullptr) + reel_fail("Comm_Free called before Comm_Init"); + delete static_cast(h->ptr); + h->ptr = nullptr; + return PCU_SUCCESS; +} + +int PCU_Comm_Self(PCU_t h) { + if (h.ptr == nullptr) + reel_fail("Comm_Self called before Comm_Init"); + return static_cast(h.ptr)->Self(); +} + +int PCU_Comm_Peers(PCU_t h) { + if (h.ptr == nullptr) + reel_fail("Comm_Peers called before Comm_Init"); + return static_cast(h.ptr)->Peers(); +} + +void PCU_Comm_Begin(PCU_t h) { + if (h.ptr == nullptr) + reel_fail("Comm_Begin called before Comm_Init"); + static_cast(h.ptr)->Begin(); +} + +/** \brief Packs data to be sent to \a to_rank. + \details This function appends the block of \a size bytes starting + at \a data to the buffer being sent to \a to_rank. + This function should be called after PCU_Comm_Start and before + PCU_Comm_Send. + */ +int PCU_Comm_Pack(PCU_t h, int to_rank, const void *data, size_t size) { + if (h.ptr == nullptr) + reel_fail("Comm_Pack called before Comm_Init"); + return static_cast(h.ptr)->Pack(to_rank, data, size); +} + +/** \brief Sends all buffers for this communication phase. + \details This function should be called by all threads in the MPI job + after calls to PCU_Comm_Pack or PCU_Comm_Write and before calls + to PCU_Comm_Listen or PCU_Comm_Read. + All buffers from this thread are sent out and receiving + may begin after this call. + */ +int PCU_Comm_Send(PCU_t h) { + if (h.ptr == nullptr) + reel_fail("Comm_Send called before Comm_Init"); + return static_cast(h.ptr)->Send(); +} + +/** \brief Tries to receive a buffer for this communication phase. + \details Either this function or PCU_Comm_Read should be called at least + once by all threads during the communication phase, after PCU_Comm_Send + is called. The result will be false if and only if the communication phase + is over and there are no more buffers to receive. + Otherwise, a buffer was received. + Its contents are retrievable through PCU_Comm_Unpack, and its metadata through + PCU_Comm_Sender and PCU_Comm_Received. + Users should unpack all data from this buffer before calling this function + again, because the previously received buffer is destroyed by the call. + */ +bool PCU_Comm_Listen(PCU_t h) { + if (h.ptr == nullptr) + reel_fail("Comm_Listen called before Comm_Init"); + return static_cast(h.ptr)->Listen(); +} + +/** \brief Returns in * \a from_rank the sender of the current received buffer. + \details This function should be called after a successful PCU_Comm_Listen. + */ +int PCU_Comm_Sender(PCU_t h) { + if (h.ptr == nullptr) + reel_fail("Comm_Sender called before Comm_Init"); + return static_cast(h.ptr)->Sender(); +} + +bool PCU_Comm_Unpacked(PCU_t h) { + if (h.ptr == nullptr) + reel_fail("Comm_Unpacked called before Comm_Init"); + return static_cast(h.ptr)->Unpacked(); +} + +/** \brief Unpacks a block of data from the current received buffer. + \details This function should be called after a successful PCU_Comm_Listen. + \a data must point to a block of memory of at least \a size bytes, into + which the next \a size bytes of the current received buffer will be written. + Subsequent calls will begin unpacking where this call left off, + so that the entire received buffer can be unpacked by a sequence of calls to + this function. + Users must ensure that there remain \a size bytes to be unpacked, + PCU_Comm_Unpacked can help with this. + */ +int PCU_Comm_Unpack(PCU_t h, void *data, size_t size) { + if (h.ptr == nullptr) + reel_fail("Comm_Unpack called before Comm_Init"); + return static_cast(h.ptr)->Unpack(data, size); +} + +void PCU_Comm_Order(PCU_t h, bool on) { + if (h.ptr == nullptr) + reel_fail("Comm_Order called before Comm_Init"); + static_cast(h.ptr)->Order(on); +} + +/** \brief Blocking barrier over all threads. */ +void PCU_Barrier(PCU_t h) { + if (h.ptr == nullptr) + reel_fail("Barrier called before Comm_Init"); + static_cast(h.ptr)->Barrier(); +} + +/** \brief Performs an Allreduce sum of double arrays. + \details This function must be called by all ranks at + the same time. \a p must point to an array of \a n doubles. + After this call, p[i] will contain the sum of all p[i]'s + given by each rank. + */ +void PCU_Add_Doubles(PCU_t h, double *p, size_t n) { + if (h.ptr == nullptr) + reel_fail("Add_Doubles called before Comm_Init"); + static_cast(h.ptr)->Add(p, n); +} + +double PCU_Add_Double(PCU_t h, double x) { + if (h.ptr == nullptr) + reel_fail("Add_Double called before Comm_Init"); + return static_cast(h.ptr)->Add(x); +} + +/** \brief Performs an Allreduce minimum of double arrays. + */ +void PCU_Min_Doubles(PCU_t h, double *p, size_t n) { + if (h.ptr == nullptr) + reel_fail("Min_Doubles called before Comm_Init"); + static_cast(h.ptr)->Min(p, n); +} + +double PCU_Min_Double(PCU_t h, double x) { + if (h.ptr == nullptr) + reel_fail("Min_Double called before Comm_Init"); + return static_cast(h.ptr)->Min(x); +} + +/** \brief Performs an Allreduce maximum of double arrays. + */ +void PCU_Max_Doubles(PCU_t h, double *p, size_t n) { + if (h.ptr == nullptr) + reel_fail("Max_Doubles called before Comm_Init"); + static_cast(h.ptr)->Max(p, n); +} + +double PCU_Max_Double(PCU_t h, double x) { + if (h.ptr == nullptr) + reel_fail("Max_Double called before Comm_Init"); + return static_cast(h.ptr)->Max(x); +} + +/** \brief Performs an Allreduce sum of integers + */ +void PCU_Add_Ints(PCU_t h, int *p, size_t n) { + if (h.ptr == nullptr) + reel_fail("Add_Ints called before Comm_Init"); + static_cast(h.ptr)->Add(p, n); +} + +int PCU_Add_Int(PCU_t h, int x) { + if (h.ptr == nullptr) + reel_fail("Add_Int called before Comm_Init"); + return static_cast(h.ptr)->Add(x); +} + +/** \brief Performs an Allreduce sum of long integers + */ +void PCU_Add_Longs(PCU_t h, long *p, size_t n) { + if (h.ptr == nullptr) + reel_fail("Add_Longs called before Comm_Init"); + static_cast(h.ptr)->Add(p, n); +} + +long PCU_Add_Long(PCU_t h, long x) { + if (h.ptr == nullptr) + reel_fail("Add_Long called before Comm_Init"); + return static_cast(h.ptr)->Add(x); +} + +/** \brief Performs an Allreduce sum of size_t unsigned integers + */ +void PCU_Add_SizeTs(PCU_t h, size_t *p, size_t n) { + if (h.ptr == nullptr) + reel_fail("Add_SizeTs called before Comm_Init"); + static_cast(h.ptr)->Add(p, n); +} + +size_t PCU_Add_SizeT(PCU_t h, size_t x) { + if (h.ptr == nullptr) + reel_fail("Add_SizeT called before Comm_Init"); + return static_cast(h.ptr)->Add(x); +} + +/** \brief Performs an Allreduce minimum of size_t unsigned integers + */ +void PCU_Min_SizeTs(PCU_t h, size_t *p, size_t n) { + if (h.ptr == nullptr) + reel_fail("Min_SizeTs called before Comm_Init"); + static_cast(h.ptr)->Min(p, n); +} + +size_t PCU_Min_SizeT(PCU_t h, size_t x) { + if (h.ptr == nullptr) + reel_fail("Min_SizeT called before Comm_Init"); + return static_cast(h.ptr)->Min(x); +} + +/** \brief Performs an Allreduce maximum of size_t unsigned integers + */ +void PCU_Max_SizeTs(PCU_t h, size_t *p, size_t n) { + if (h.ptr == nullptr) + reel_fail("Max_SizeTs called before Comm_Init"); + static_cast(h.ptr)->Max(p, n); +} + +size_t PCU_Max_SizeT(PCU_t h, size_t x) { + if (h.ptr == nullptr) + reel_fail("Max_SizeT called before Comm_Init"); + return static_cast(h.ptr)->Max(x); +} + +/** \brief Performs an exclusive prefix sum of integer arrays. + \details This function must be called by all ranks at + the same time. \a p must point to an array of \a n integers. + After this call, p[i] will contain the sum of all p[i]'s + given by ranks lower than the calling rank. + */ +void PCU_Exscan_Ints(PCU_t h, int *p, size_t n) { + if (h.ptr == nullptr) + reel_fail("Exscan_Ints called before Comm_Init"); + static_cast(h.ptr)->Exscan(p, n); +} + +int PCU_Exscan_Int(PCU_t h, int x) { + if (h.ptr == nullptr) + reel_fail("Exscan_Int called before Comm_Init"); + return static_cast(h.ptr)->Exscan(x); +} + +/** \brief See PCU_Exscan_Ints */ +void PCU_Exscan_Longs(PCU_t h, long *p, size_t n) { + if (h.ptr == nullptr) + reel_fail("Exscan_Longs called before Comm_Init"); + static_cast(h.ptr)->Exscan(p, n); +} + +long PCU_Exscan_Long(PCU_t h, long x) { + if (h.ptr == nullptr) + reel_fail("Exscan_Long called before Comm_Init"); + return static_cast(h.ptr)->Exscan(x); +} + +/** \brief Performs an Allreduce minimum of int arrays. + */ +void PCU_Min_Ints(PCU_t h, int *p, size_t n) { + if (h.ptr == nullptr) + reel_fail("Min_Ints called before Comm_Init"); + static_cast(h.ptr)->Min(p, n); +} + +int PCU_Min_Int(PCU_t h, int x) { + if (h.ptr == nullptr) + reel_fail("Min_Int called before Comm_Init"); + return static_cast(h.ptr)->Min(x); +} + +/** \brief Performs an Allreduce maximum of int arrays. + */ +void PCU_Max_Ints(PCU_t h, int *p, size_t n) { + if (h.ptr == nullptr) + reel_fail("Max_Ints called before Comm_Init"); + static_cast(h.ptr)->Max(p, n); +} + +int PCU_Max_Int(PCU_t h, int x) { + if (h.ptr == nullptr) + reel_fail("Max_Int called before Comm_Init"); + return static_cast(h.ptr)->Max(x); +} +/** \brief Performs an Allreduce maximum of long arrays. + */ +void PCU_Max_Longs(PCU_t h, long *p, size_t n) { + if (h.ptr == nullptr) + reel_fail("Max_Longs called before Comm_Init"); + static_cast(h.ptr)->Max(p, n); +} + +long PCU_Max_Long(PCU_t h, long x) { + if (h.ptr == nullptr) + reel_fail("Max_Long called before Comm_Init"); + return static_cast(h.ptr)->Max(x); +} + +/** \brief Performs a parallel logical OR reduction + */ +int PCU_Or(PCU_t h, int c) { + if (h.ptr == nullptr) + reel_fail("Or called before Comm_Init"); + return static_cast(h.ptr)->Or(c); +} + +/** \brief Performs a parallel logical AND reduction + */ +int PCU_And(PCU_t h, int c) { + if (h.ptr == nullptr) + reel_fail("And called before Comm_Init"); + return static_cast(h.ptr)->And(c); +} + +/** \brief Returns the unique rank of the calling process. + */ +int PCU_Proc_Self(PCU_t h) { + if (h.ptr == nullptr) + reel_fail("Proc_Self called before Comm_Init"); + return static_cast(h.ptr)->Self(); +} + +/** \brief Returns the number of processes. + */ +int PCU_Proc_Peers(PCU_t h) { + if (h.ptr == nullptr) + reel_fail("Proc_Peers called before Comm_Init"); + return static_cast(h.ptr)->Peers(); +} + +/** \brief Similar to PCU_Comm_Self, returns the rank as an argument. + */ +int PCU_Comm_Rank(PCU_t h, int *rank) { + if (h.ptr == nullptr) + reel_fail("Comm_Rank called before Comm_Init"); + *rank = static_cast(h.ptr)->Self(); + return PCU_SUCCESS; +} + +/** \brief Similar to PCU_Comm_Peers, returns the size as an argument. */ +int PCU_Comm_Size(PCU_t h, int *size) { + if (h.ptr == nullptr) + reel_fail("Comm_Size called before Comm_Init"); + *size = static_cast(h.ptr)->Peers(); + return PCU_SUCCESS; +} + +/** \brief Returns true iff PCU has been initialized */ +bool PCU_Comm_Initialized(PCU_t h) { return h.ptr != nullptr; } + + + +/** \brief Returns in * \a size the number of bytes being sent to \a to_rank. + \details Returns the size of the buffer being sent to \a to_rank. + This function should be called after PCU_Comm_Start and before + PCU_Comm_Send. + */ +int PCU_Comm_Packed(PCU_t h, int to_rank, size_t *size) { + if (h.ptr == nullptr) + reel_fail("Comm_Packed called before Comm_Init"); + return static_cast(h.ptr)->Packed(to_rank, size); +} + +/** \brief Packs a message to be sent to \a to_rank. + \details This function packs a message into the buffer being sent + to \a to_rank. + Messages packed by this function can be received using the function + PCU_Comm_Read. + This function should be called after PCU_Comm_Start and before + PCU_Comm_Send. + If this function is used, PCU_Comm_Pack should not be used. + */ +int PCU_Comm_Write(PCU_t h, int to_rank, const void *data, size_t size) { + if (h.ptr == nullptr) + reel_fail("Comm_Write called before Comm_Init"); + return static_cast(h.ptr)->Write(to_rank, data, size); +} + +/** \brief Convenience wrapper over Listen and Unpacked */ +bool PCU_Comm_Receive(PCU_t h) { + if (h.ptr == nullptr) + reel_fail("Comm_Receive called before Comm_Init"); + return static_cast(h.ptr)->Receive(); +} + +/** \brief Receives a message for this communication phase. + \details This function tries to receive a message packed by + PCU_Comm_Write. + If a the communication phase is over and there are no more + messages to receive, this function returns false. + Otherwise, * \a from_rank will be the rank which sent the message, + * \a data will point to the start of the message data, and + * \a size will be the number of bytes of message data. + If this function is used, PCU_Comm_Receive should not be used. + Note that the address * \a data points into a PCU buffer, so + it is strongly recommended that this data be read and not modified. + */ +bool PCU_Comm_Read(PCU_t h, int *from_rank, void **data, size_t *size) { + if (h.ptr == nullptr) + reel_fail("Comm_Read called before Comm_Init"); + return static_cast(h.ptr)->Read(from_rank, data, size); +} + +void PCU_Debug_Open(PCU_t h) { + if (h.ptr == nullptr) + reel_fail("Debug_Open called before Comm_Init"); + static_cast(h.ptr)->DebugOpen(); +} + +/** \brief like fprintf, contents go to debugN.txt */ +void PCU_Debug_Print(PCU_t h, const char *format, ...) { + if (h.ptr == nullptr) + reel_fail("Debug_Print called before Comm_Init"); + va_list arglist; + va_start(arglist, format); + static_cast(h.ptr)->DebugPrint(format, arglist); + va_end(arglist); +} + +/** \brief Similar to PCU_Comm_Sender, returns the rank as an argument. */ +int PCU_Comm_From(PCU_t h, int *from_rank) { + if (h.ptr == nullptr) + reel_fail("Comm_From called before Comm_Init"); + return static_cast(h.ptr)->From(from_rank); +} + +/** \brief Returns in * \a size the bytes in the current received buffer + \details This function should be called after a successful PCU_Comm_Receive. + The size returned will be the total received size regardless of how + much unpacking has been done. + */ +int PCU_Comm_Received(PCU_t h, size_t *size) { + if (h.ptr == nullptr) + reel_fail("Comm_Received called before Comm_Init"); + return static_cast(h.ptr)->Received(size); +} + +/** \brief Extracts a block of data from the current received buffer. + \details This function should be called after a successful PCU_Comm_Receive. + The next \a size bytes of the current received buffer are unpacked, + and an internal pointer to that data is returned. + The returned pointer must not be freed by the user. + */ +void *PCU_Comm_Extract(PCU_t h, size_t size) { + if (h.ptr == nullptr) + reel_fail("Comm_Extract called before Comm_Init"); + return static_cast(h.ptr)->Extract(size); +} + +/** \brief Reinitializes PCU with a new MPI communicator. + \details All of PCU's logic is based off two duplicates + of this communicator, so you can safely get PCU to act + on sub-groups of processes using this function. + This call should be collective over all processes + in the previous communicator. This is a very heavy weight function + and should be used sparingly. + */ +void PCU_Switch_Comm(PCU_t h, MPI_Comm new_comm) { + if (h.ptr == nullptr) + reel_fail("Switch_Comm called before Comm_Init"); + static_cast(h.ptr)->SwitchMPIComm(new_comm); +} + +/** \brief Return the current MPI communicator + \details Returns the communicator given to the + most recent PCU_Switch_Comm call, or MPI_COMM_WORLD + otherwise. + */ +MPI_Comm PCU_Get_Comm(PCU_t h) { + if (h.ptr == nullptr) + reel_fail("Get_Comm called before Comm_Init"); + return static_cast(h.ptr)->GetMPIComm(); +} + +/** \brief Return the time in seconds since some time in the past + */ +double PCU_Time(void) { return pcu::Time(); } + +void PCU_Protect(void) { return pcu::Protect(); } + +double PCU_GetMem(void) { return pcu::GetMem(); } + + +} \ No newline at end of file diff --git a/pcu/pcu_coll.c b/pcu/pcu_coll.c index 7bdf8a221..01c89445f 100644 --- a/pcu/pcu_coll.c +++ b/pcu/pcu_coll.c @@ -8,7 +8,6 @@ *******************************************************************************/ #include "pcu_coll.h" -#include "pcu_pmpi.h" #include "reel.h" #include @@ -34,132 +33,33 @@ void pcu_merge_assign(void* local, void* incoming, size_t size) memcpy(local,incoming,size); } -void pcu_add_doubles(void* local, void* incoming, size_t size) -{ - double* a = local; - double* b= incoming; - size_t n = size/sizeof(double); - for (size_t i=0; i < n; ++i) - a[i] += b[i]; -} - -void pcu_max_doubles(void* local, void* incoming, size_t size) -{ - double* a = local; - double* b= incoming; - size_t n = size/sizeof(double); - for (size_t i=0; i < n; ++i) - a[i] = MAX(a[i],b[i]); -} - -void pcu_min_doubles(void* local, void* incoming, size_t size) -{ - double* a = local; - double* b= incoming; - size_t n = size/sizeof(double); - for (size_t i=0; i < n; ++i) - a[i] = MIN(a[i],b[i]); -} - -void pcu_add_ints(void* local, void* incoming, size_t size) -{ - int* a = local; - int* b= incoming; - size_t n = size/sizeof(int); - for (size_t i=0; i < n; ++i) - a[i] += b[i]; -} - -void pcu_min_ints(void* local, void* incoming, size_t size) -{ - int* a = local; - int* b= incoming; - size_t n = size/sizeof(int); - for (size_t i=0; i < n; ++i) - a[i] = MIN(a[i],b[i]); -} - -void pcu_max_ints(void* local, void* incoming, size_t size) -{ - int* a = local; - int* b= incoming; - size_t n = size/sizeof(int); - for (size_t i=0; i < n; ++i) - a[i] = MAX(a[i],b[i]); -} - -void pcu_max_longs(void* local, void* incoming, size_t size) -{ - long* a = local; - long* b= incoming; - size_t n = size/sizeof(long); - for (size_t i=0; i < n; ++i) - a[i] = MAX(a[i],b[i]); -} - -void pcu_min_sizets(void* local, void* incoming, size_t size) -{ - size_t* a = local; - size_t* b = incoming; - size_t n = size/sizeof(size_t); - for (size_t i=0; i < n; ++i) - a[i] = MIN(a[i],b[i]); -} - -void pcu_max_sizets(void* local, void* incoming, size_t size) -{ - size_t* a = local; - size_t* b = incoming; - size_t n = size/sizeof(size_t); - for (size_t i=0; i < n; ++i) - a[i] = MAX(a[i],b[i]); -} - -void pcu_add_sizets(void* local, void* incoming, size_t size) -{ - size_t* a = local; - size_t* b = incoming; - size_t n = size/sizeof(size_t); - for (size_t i=0; i < n; ++i) - a[i] += b[i]; -} - -void pcu_add_longs(void* local, void* incoming, size_t size) -{ - long* a = local; - long* b= incoming; - size_t n = size/sizeof(long); - for (size_t i=0; i < n; ++i) - a[i] += b[i]; -} - /* initiates non-blocking calls for this communication step */ -static void begin_coll_step(pcu_coll* c) +static void begin_coll_step(pcu_mpi_t* mpi, pcu_coll* c) { - int action = c->pattern->action(c->bit); + int action = c->pattern->action(mpi, c->bit); if (action == pcu_coll_idle) return; - c->message.peer = c->pattern->peer(c->bit); + c->message.peer = c->pattern->peer(mpi, c->bit); if (action == pcu_coll_send) - pcu_mpi_send(&(c->message),pcu_coll_comm); + pcu_mpi_send(mpi, &(c->message),mpi->coll_comm); } /* tries to complete this communication step. Returns false if communication is not done, otherwise wraps up communication, merges if necessary, and returns true */ -static bool end_coll_step(pcu_coll* c) +static bool end_coll_step(pcu_mpi_t* mpi, pcu_coll* c) { - int action = c->pattern->action(c->bit); + int action = c->pattern->action(mpi, c->bit); if (action == pcu_coll_idle) return true; if (action == pcu_coll_send) - return pcu_mpi_done(&(c->message)); + return pcu_mpi_done(mpi, &(c->message)); pcu_message incoming; pcu_make_message(&incoming); - incoming.peer = c->pattern->peer(c->bit); - if ( ! pcu_mpi_receive(&incoming,pcu_coll_comm)) + incoming.peer = c->pattern->peer(mpi, c->bit); + if ( ! pcu_mpi_receive(mpi, &incoming,mpi->coll_comm)) return false; if (c->message.buffer.size != incoming.buffer.size) reel_fail("PCU unexpected incoming message.\n" @@ -169,8 +69,10 @@ static bool end_coll_step(pcu_coll* c) return true; } -void pcu_make_coll(pcu_coll* c, pcu_pattern* p, pcu_merge* m) +void pcu_make_coll(pcu_mpi_t* mpi, pcu_coll* c, pcu_pattern* p, pcu_merge* m) { + // silence warning + (void)mpi; c->pattern = p; c->merge = m; } @@ -194,28 +96,28 @@ void pcu_make_coll(pcu_coll* c, pcu_pattern* p, pcu_merge* m) /* begins a non-blocking collective. The collective operation should be set with pcu_make_coll first. data[0..size] is the input/output local data */ -void pcu_begin_coll(pcu_coll* c, void* data, size_t size) +void pcu_begin_coll(pcu_mpi_t* mpi, pcu_coll* c, void* data, size_t size) { pcu_set_buffer(&(c->message.buffer),data,size); - c->bit = c->pattern->begin_bit(); - if (c->pattern->end_bit(c->bit)) + c->bit = c->pattern->begin_bit(mpi); + if (c->pattern->end_bit(mpi, c->bit)) return; - begin_coll_step(c); + begin_coll_step(mpi, c); } /* makes progress on a collective operation started by pcu_begin_coll. returns false if its done. */ -bool pcu_progress_coll(pcu_coll* c) +bool pcu_progress_coll(pcu_mpi_t* mpi, pcu_coll* c) { - if (c->pattern->end_bit(c->bit)) + if (c->pattern->end_bit(mpi, c->bit)) return false; - if (end_coll_step(c)) + if (end_coll_step(mpi, c)) { - c->bit = c->pattern->shift(c->bit); - if (c->pattern->end_bit(c->bit)) + c->bit = c->pattern->shift(mpi, c->bit); + if (c->pattern->end_bit(mpi, c->bit)) return false; - begin_coll_step(c); + begin_coll_step(mpi, c); } return true; } @@ -224,35 +126,39 @@ bool pcu_progress_coll(pcu_coll* c) then odd multiples of 2 into even ones, etc... until rank 0 has all inputs merged */ -static int reduce_begin_bit(void) +static int reduce_begin_bit(pcu_mpi_t* mpi) { + // silence warning + (void)mpi; return 1; } -static bool reduce_end_bit(int bit) +static bool reduce_end_bit(pcu_mpi_t* mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if (rank==0) - return bit >= pcu_mpi_size(); + return bit >= pcu_mpi_size(mpi); return (bit>>1) & rank; } -static int reduce_peer(int bit) +static int reduce_peer(pcu_mpi_t* mpi, int bit) { - return pcu_mpi_rank() ^ bit; + return pcu_mpi_rank(mpi) ^ bit; } -static int reduce_action(int bit) +static int reduce_action(pcu_mpi_t* mpi, int bit) { - if (reduce_peer(bit) >= pcu_mpi_size()) + if (reduce_peer(mpi, bit) >= pcu_mpi_size(mpi)) return pcu_coll_idle; - if (bit & pcu_mpi_rank()) + if (bit & pcu_mpi_rank(mpi)) return pcu_coll_send; return pcu_coll_recv; } -static int reduce_shift(int bit) +static int reduce_shift(pcu_mpi_t* mpi, int bit) { + // silence warning + (void)mpi; return bit << 1; } @@ -269,37 +175,41 @@ static pcu_pattern reduce = the pattern runs backwards and send/recv are flipped. */ -static int bcast_begin_bit(void) +static int bcast_begin_bit(pcu_mpi_t* mpi) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if (rank == 0) - return 1 << ceil_log2(pcu_mpi_size()); + return 1 << ceil_log2(pcu_mpi_size(mpi)); int bit = 1; while ( ! (bit & rank)) bit <<= 1; return bit; } -static bool bcast_end_bit(int bit) +static bool bcast_end_bit(pcu_mpi_t * mpi, int bit) { + // silence warning + (void)mpi; return bit == 0; } -static int bcast_peer(int bit) +static int bcast_peer(pcu_mpi_t * mpi, int bit) { - return pcu_mpi_rank() ^ bit; + return pcu_mpi_rank(mpi) ^ bit; } -static int bcast_action(int bit) +static int bcast_action(pcu_mpi_t* mpi, int bit) { - if (bcast_peer(bit) >= pcu_mpi_size()) + if (bcast_peer(mpi, bit) >= pcu_mpi_size(mpi)) return pcu_coll_idle; - if (bit & pcu_mpi_rank()) + if (bit & pcu_mpi_rank(mpi)) return pcu_coll_recv; return pcu_coll_send; } -static int bcast_shift(int bit) +static int bcast_shift(pcu_mpi_t * mpi, int bit) { + // silence warning + (void)mpi; return bit >> 1; } @@ -319,14 +229,16 @@ static pcu_pattern bcast = "Parallel Prefix (Scan) Algorithms for MPI". */ -static int scan_up_begin_bit(void) +static int scan_up_begin_bit(pcu_mpi_t* mpi) { + // silence warning + (void)mpi; return 1; } -static bool scan_up_end_bit(int bit) +static bool scan_up_end_bit(pcu_mpi_t* mpi, int bit) { - return bit == (1 << floor_log2(pcu_mpi_size())); + return bit == (1 << floor_log2(pcu_mpi_size(mpi))); } static bool scan_up_could_receive(int rank, int bit) @@ -345,35 +257,37 @@ static int scan_up_receiver_for(int rank, int bit) return rank + bit; } -static int scan_up_action(int bit) +static int scan_up_action(pcu_mpi_t* mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if ((scan_up_could_receive(rank,bit))&& (0 <= scan_up_sender_for(rank,bit))) return pcu_coll_recv; int receiver = scan_up_receiver_for(rank,bit); - if ((receiver < pcu_mpi_size())&& + if ((receiver < pcu_mpi_size(mpi))&& (scan_up_could_receive(receiver,bit))) return pcu_coll_send; return pcu_coll_idle; } -static int scan_up_peer(int bit) +static int scan_up_peer(pcu_mpi_t* mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); int sender = scan_up_sender_for(rank,bit); if ((scan_up_could_receive(rank,bit))&& (0 <= sender)) return sender; int receiver = scan_up_receiver_for(rank,bit); - if ((receiver < pcu_mpi_size())&& + if ((receiver < pcu_mpi_size(mpi))&& (scan_up_could_receive(receiver,bit))) return receiver; return -1; } -static int scan_up_shift(int bit) +static int scan_up_shift(pcu_mpi_t* mpi, int bit) { + // silence warning + (void)mpi; return bit << 1; } @@ -386,13 +300,15 @@ static pcu_pattern scan_up = .shift = scan_up_shift, }; -static int scan_down_begin_bit(void) +static int scan_down_begin_bit(pcu_mpi_t* mpi) { - return 1 << floor_log2(pcu_mpi_size()); + return 1 << floor_log2(pcu_mpi_size(mpi)); } -static bool scan_down_end_bit(int bit) +static bool scan_down_end_bit(pcu_mpi_t* mpi, int bit) { + // silence warning + (void)mpi; return bit == 1; } @@ -412,11 +328,11 @@ static int scan_down_sender_for(int rank, int bit) return rank - (bit >> 1); } -static int scan_down_action(int bit) +static int scan_down_action(pcu_mpi_t * mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if ((scan_down_could_send(rank,bit))&& - (scan_down_receiver_for(rank,bit) < pcu_mpi_size())) + (scan_down_receiver_for(rank,bit) < pcu_mpi_size(mpi))) return pcu_coll_send; int sender = scan_down_sender_for(rank,bit); if ((0 <= sender)&& @@ -425,13 +341,13 @@ static int scan_down_action(int bit) return pcu_coll_idle; } -static int scan_down_peer(int bit) +static int scan_down_peer(pcu_mpi_t * mpi, int bit) { - int rank = pcu_mpi_rank(); + int rank = pcu_mpi_rank(mpi); if (scan_down_could_send(rank,bit)) { int receiver = scan_down_receiver_for(rank,bit); - if (receiver < pcu_mpi_size()) + if (receiver < pcu_mpi_size(mpi)) return receiver; } int sender = scan_down_sender_for(rank,bit); @@ -441,8 +357,10 @@ static int scan_down_peer(int bit) return -1; } -static int scan_down_shift(int bit) +static int scan_down_shift(pcu_mpi_t* mpi, int bit) { + // silence warning + (void)mpi; return bit >> 1; } @@ -455,59 +373,61 @@ static pcu_pattern scan_down = .shift = scan_down_shift, }; -void pcu_reduce(pcu_coll* c, pcu_merge* m, void* data, size_t size) +void pcu_reduce(pcu_mpi_t* mpi, pcu_coll* c, pcu_merge* m, void* data, size_t size) { - pcu_make_coll(c,&reduce,m); - pcu_begin_coll(c,data,size); - while(pcu_progress_coll(c)); + pcu_make_coll(mpi, c,&reduce,m); + pcu_begin_coll(mpi, c,data,size); + while(pcu_progress_coll(mpi, c)); } -void pcu_bcast(pcu_coll* c, void* data, size_t size) +void pcu_bcast(pcu_mpi_t * mpi, pcu_coll* c, void* data, size_t size) { - pcu_make_coll(c,&bcast,pcu_merge_assign); - pcu_begin_coll(c,data,size); - while(pcu_progress_coll(c)); + pcu_make_coll(mpi, c,&bcast,pcu_merge_assign); + pcu_begin_coll(mpi, c,data,size); + while(pcu_progress_coll(mpi, c)); } -void pcu_allreduce(pcu_coll* c, pcu_merge* m, void* data, size_t size) +void pcu_allreduce(pcu_mpi_t* mpi, pcu_coll* c, pcu_merge* m, void* data, size_t size) { - pcu_reduce(c,m,data,size); - pcu_bcast(c,data,size); + pcu_reduce(mpi, c,m,data,size); + pcu_bcast(mpi, c,data,size); } -void pcu_scan(pcu_coll* c, pcu_merge* m, void* data, size_t size) +void pcu_scan(pcu_mpi_t* mpi, pcu_coll* c, pcu_merge* m, void* data, size_t size) { - pcu_make_coll(c,&scan_up,m); - pcu_begin_coll(c,data,size); - while(pcu_progress_coll(c)); - pcu_make_coll(c,&scan_down,m); - pcu_begin_coll(c,data,size); - while(pcu_progress_coll(c)); + pcu_make_coll(mpi, c,&scan_up,m); + pcu_begin_coll(mpi, c,data,size); + while(pcu_progress_coll(mpi, c)); + pcu_make_coll(mpi, c,&scan_down,m); + pcu_begin_coll(mpi, c,data,size); + while(pcu_progress_coll(mpi, c)); } /* a barrier is just an allreduce of nothing in particular */ -void pcu_begin_barrier(pcu_coll* c) +void pcu_begin_barrier(pcu_mpi_t* mpi, pcu_coll* c) { - pcu_make_coll(c,&reduce,pcu_merge_assign); - pcu_begin_coll(c,NULL,0); + pcu_make_coll(mpi, c,&reduce,pcu_merge_assign); + pcu_begin_coll(mpi, c,NULL,0); } -bool pcu_barrier_done(pcu_coll* c) +bool pcu_barrier_done(pcu_mpi_t* mpi, pcu_coll* c) { if (c->pattern == &reduce) - if ( ! pcu_progress_coll(c)) + if ( ! pcu_progress_coll(mpi, c)) { - pcu_make_coll(c,&bcast,pcu_merge_assign); - pcu_begin_coll(c,c->message.buffer.start,c->message.buffer.size); + pcu_make_coll(mpi, c,&bcast,pcu_merge_assign); + pcu_begin_coll(mpi, c,c->message.buffer.start,c->message.buffer.size); } if (c->pattern == &bcast) - if ( ! pcu_progress_coll(c)) + if ( ! pcu_progress_coll(mpi,c)) return true; return false; } -void pcu_barrier(pcu_coll* c) +void pcu_barrier(pcu_mpi_t* mpi, pcu_coll* c) { - pcu_begin_barrier(c); - while( ! pcu_barrier_done(c)); + // silence warning + (void)mpi; + pcu_begin_barrier(mpi, c); + while( ! pcu_barrier_done(mpi, c)); } diff --git a/pcu/pcu_coll.h b/pcu/pcu_coll.h index 11f204ee8..f69bf7a08 100644 --- a/pcu/pcu_coll.h +++ b/pcu/pcu_coll.h @@ -12,6 +12,10 @@ #include "pcu_mpi.h" +#ifdef __cplusplus +extern "C" { +#endif + /* The PCU Collectives system (pcu_coll for short) implements non-blocking collective operations based loosely on binary-tree or binomial communication patterns. @@ -27,17 +31,6 @@ typedef void pcu_merge(void* local, void* incoming, size_t size); void pcu_merge_assign(void* local, void* incoming, size_t size); -void pcu_add_doubles(void* local, void* incoming, size_t size); -void pcu_max_doubles(void* local, void* incoming, size_t size); -void pcu_min_doubles(void* local, void* incoming, size_t size); -void pcu_add_ints(void* local, void* incoming, size_t size); -void pcu_min_ints(void* local, void* incoming, size_t size); -void pcu_max_ints(void* local, void* incoming, size_t size); -void pcu_max_longs(void* local, void* incoming, size_t size); -void pcu_add_longs(void* local, void* incoming, size_t size); -void pcu_add_sizets(void* local, void* incoming, size_t size); -void pcu_min_sizets(void* local, void* incoming, size_t size); -void pcu_max_sizets(void* local, void* incoming, size_t size); /* Enumerated actions that a rank takes during one step of the communication pattern */ @@ -57,11 +50,11 @@ enum */ typedef struct { - int (*begin_bit)(void); //initialize state bit - bool (*end_bit)(int bit); //return true if bit is one past the last - int (*action)(int bit); //return action enum for this step - int (*peer)(int bit); //return the peer to communicate with - int (*shift)(int bit); //shift the bit up or down + int (*begin_bit)(pcu_mpi_t*); //initialize state bit + bool (*end_bit)(pcu_mpi_t*, int bit); //return true if bit is one past the last + int (*action)(pcu_mpi_t*, int bit); //return action enum for this step + int (*peer)(pcu_mpi_t*, int bit); //return the peer to communicate with + int (*shift)(pcu_mpi_t*, int bit); //shift the bit up or down } pcu_pattern; /* The pcu_coll object stores the state of a non-blocking @@ -74,18 +67,21 @@ typedef struct int bit; //pattern's state bit } pcu_coll; -void pcu_make_coll(pcu_coll* c, pcu_pattern* p, pcu_merge* m); -void pcu_begin_coll(pcu_coll* c, void* data, size_t size); +void pcu_make_coll(pcu_mpi_t *, pcu_coll* c, pcu_pattern* p, pcu_merge* m); +void pcu_begin_coll(pcu_mpi_t *, pcu_coll* c, void* data, size_t size); //returns false when done -bool pcu_progress_coll(pcu_coll* c); +bool pcu_progress_coll(pcu_mpi_t* mpi, pcu_coll* c); -void pcu_reduce(pcu_coll* c, pcu_merge* m, void* data, size_t size); -void pcu_bcast(pcu_coll* c, void* data, size_t size); -void pcu_allreduce(pcu_coll* c, pcu_merge* m, void* data, size_t size); -void pcu_scan(pcu_coll* c, pcu_merge* m, void* data, size_t size); +void pcu_reduce(pcu_mpi_t*, pcu_coll* c, pcu_merge* m, void* data, size_t size); +void pcu_bcast(pcu_mpi_t*, pcu_coll* c, void* data, size_t size); +void pcu_allreduce(pcu_mpi_t*, pcu_coll* c, pcu_merge* m, void* data, size_t size); +void pcu_scan(pcu_mpi_t*, pcu_coll* c, pcu_merge* m, void* data, size_t size); -void pcu_begin_barrier(pcu_coll* c); -bool pcu_barrier_done(pcu_coll* c); -void pcu_barrier(pcu_coll* c); +void pcu_begin_barrier(pcu_mpi_t*,pcu_coll* c); +bool pcu_barrier_done(pcu_mpi_t*, pcu_coll* c); +void pcu_barrier(pcu_mpi_t*, pcu_coll* c); +#ifdef __cplusplus +} +#endif #endif //PCU_COLL_H diff --git a/pcu/pcu_defines.h b/pcu/pcu_defines.h new file mode 100644 index 000000000..acf8861e5 --- /dev/null +++ b/pcu/pcu_defines.h @@ -0,0 +1,25 @@ +#ifndef SCOREC_PCU_PCU_DEFINES_H +#define SCOREC_PCU_PCU_DEFINES_H + +#define PCU_SUCCESS 0 +#define PCU_FAILURE -1 +#ifdef __GNUC__ +#define PCU_FORMAT_ATTRIBUTE(...) \ + __attribute__((format(printf, ##__VA_ARGS__))); +#else +#define PCU_FORMAT_ATTRIBUTE(format, ...) +#endif + +#ifdef __cplusplus +extern "C"{ +#endif + +struct PCU_t { + void* ptr; +}; + +#ifdef __cplusplus +} +#endif + +#endif // SCOREC_PCU_PCU_DEFINES_H \ No newline at end of file diff --git a/pcu/pcu_io.c b/pcu/pcu_io.c index a706f6155..5d698984e 100644 --- a/pcu/pcu_io.c +++ b/pcu/pcu_io.c @@ -8,16 +8,16 @@ *******************************************************************************/ #include "pcu_io.h" +#include "PCU_C.h" #include "noto_malloc.h" +#include "pcu_buffer.h" +#include "pcu_util.h" #include "reel.h" -#include "pcu_mpi.h" -#include "PCU.h" +#include #include -#include #include -#include "pcu_util.h" +#include #include -#include #ifdef PCU_BZIP #include @@ -158,18 +158,18 @@ static void close_compressed(pcu_file* pf) * Ideally, the filesystem handles the load and this can be * removed. */ -FILE* pcu_group_open(const char* path, bool write) { +FILE* pcu_group_open(PCU_t h, const char* path, bool write) { FILE* fp = NULL; - const int rank = PCU_Comm_Self(); + const int rank = PCU_Comm_Self(h); const char* mode = write ? "w" : "r"; const int group_size = 4096; - const int q = PCU_Comm_Peers()/group_size; - const int r = PCU_Comm_Peers()%group_size; + const int q = PCU_Comm_Peers(h)/group_size; + const int r = PCU_Comm_Peers(h)%group_size; const int groups = q + ( r > 0 ); if(!rank && groups > 1) { fprintf(stderr, "pcu peers %d max group size %d posix groups %d\n", - PCU_Comm_Peers(), group_size, groups); + PCU_Comm_Peers(h), group_size, groups); } for(int i=0; icompress = compress; pf->write = write; - pf->f = pcu_group_open(name, write); + pf->f = pcu_group_open(h, name, write); if (!pf->f) { perror("pcu_fopen"); reel_fail("pcu_fopen couldn't open \"%s\"", name); @@ -355,13 +355,14 @@ void pcu_write_string (pcu_file * f, const char * p) pcu_write (f, p, len + 1); } -FILE* pcu_open_parallel(const char* prefix, const char* ext) +FILE* pcu_open_parallel(PCU_t h, const char* prefix, const char* ext) { //max_rank_chars = strlen("4294967296"), 4294967296 = 2^32 ~= INT_MAX static const size_t max_rank_chars = 10; size_t path_size = strlen(prefix) + max_rank_chars + strlen(ext) + 1; char* path = noto_malloc(path_size); - int rank = pcu_mpi_rank(); + int rank; + PCU_Comm_Rank(h, &rank); snprintf(path,path_size,"%s%d.%s",prefix,rank,ext); FILE* file = fopen(path, "w"); noto_free(path); diff --git a/pcu/pcu_io.h b/pcu/pcu_io.h index 1ba0f48d7..ee223b082 100644 --- a/pcu/pcu_io.h +++ b/pcu/pcu_io.h @@ -20,8 +20,9 @@ extern "C" { #endif struct pcu_file; +typedef struct PCU_t PCU_t; -struct pcu_file* pcu_fopen(const char* path, bool write, bool compress); +struct pcu_file* pcu_fopen(PCU_t h, const char* path, bool write, bool compress); void pcu_fclose (struct pcu_file * pf); void pcu_read(struct pcu_file* f, char* p, size_t n); void pcu_write(struct pcu_file* f, const char* p, size_t n); @@ -34,8 +35,8 @@ void pcu_write_doubles(struct pcu_file* f, double* p, size_t n); void pcu_read_string(struct pcu_file* f, char** p); void pcu_write_string(struct pcu_file* f, const char* p); -FILE* pcu_open_parallel(const char* prefix, const char* ext); -FILE* pcu_group_open(const char* path, bool write); +FILE* pcu_open_parallel(PCU_t h, const char* prefix, const char* ext); +FILE* pcu_group_open(PCU_t h, const char* path, bool write); void pcu_swap_doubles(double* p, size_t n); void pcu_swap_unsigneds(unsigned* p, size_t n); diff --git a/pcu/pcu_mem.c b/pcu/pcu_mem.c index f71f75cab..068dc99e6 100644 --- a/pcu/pcu_mem.c +++ b/pcu/pcu_mem.c @@ -7,7 +7,16 @@ BSD license as described in the LICENSE file in the top-level directory. *******************************************************************************/ -#include +#include +#ifdef __cplusplus +#include +#include +extern "C" { +#else +#include +#include +#include +#endif #if defined(__APPLE__) @@ -39,7 +48,7 @@ #endif -double PCU_GetMem() { +double pcu_get_mem() { const double M = 1024*1024; #if defined(__APPLE__) bool resident = true; diff --git a/pcu/pcu_mem.h b/pcu/pcu_mem.h new file mode 100644 index 000000000..50f7419b6 --- /dev/null +++ b/pcu/pcu_mem.h @@ -0,0 +1,13 @@ +#ifndef SCOREC_PCU_PCU_MEM_H +#define SCOREC_PCU_PCU_MEM_H + +#ifdef __cplusplus +extern "C" { +#endif + +double pcu_get_mem(); + +#ifdef __cplusplus +} +#endif +#endif // SCOREC_PCU_PCU_MEM_H diff --git a/pcu/pcu_mpi.c b/pcu/pcu_mpi.c index 75009d277..5681d0e90 100644 --- a/pcu/pcu_mpi.c +++ b/pcu/pcu_mpi.c @@ -9,8 +9,7 @@ *******************************************************************************/ #include "pcu_mpi.h" #include "pcu_util.h" - -static pcu_mpi* global_mpi; +#include "pcu_pmpi.h" void pcu_make_message(pcu_message* m) { @@ -22,47 +21,44 @@ void pcu_free_message(pcu_message* m) pcu_free_buffer(&(m->buffer)); } -void pcu_set_mpi(pcu_mpi* m) +int pcu_mpi_size(const pcu_mpi_t* self) { - global_mpi = m; + return pcu_pmpi_size(self); } -pcu_mpi* pcu_get_mpi(void) +int pcu_mpi_rank(const pcu_mpi_t* self) { - return global_mpi; + return pcu_pmpi_rank(self); } -int pcu_mpi_size(void) -{ - return global_mpi->size(); -} - -int pcu_mpi_rank(void) -{ - return global_mpi->rank(); -} - -static void check_rank(int rank) +static void check_rank(const pcu_mpi_t* self, int rank) { (void)rank; PCU_ALWAYS_ASSERT(0 <= rank); - PCU_ALWAYS_ASSERT(rank < pcu_mpi_size()); + PCU_ALWAYS_ASSERT(rank < pcu_mpi_size(self)); } -void pcu_mpi_send(pcu_message* m, MPI_Comm comm) +void pcu_mpi_send(const pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { - check_rank(m->peer); - global_mpi->send(m,comm); + check_rank(self, m->peer); + PCU_ALWAYS_ASSERT(comm == self->user_comm || comm == self->coll_comm); + pcu_pmpi_send(self, m, comm); } -bool pcu_mpi_done(pcu_message* m) +bool pcu_mpi_done(const pcu_mpi_t* self, pcu_message* m) { - return global_mpi->done(m); + return pcu_pmpi_done(self, m); } -bool pcu_mpi_receive(pcu_message* m, MPI_Comm comm) +bool pcu_mpi_receive(const pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { if (m->peer != MPI_ANY_SOURCE) - check_rank(m->peer); - return global_mpi->receive(m,comm); + check_rank(self, m->peer); + return pcu_pmpi_receive(self, m, comm); +} +void pcu_mpi_init(MPI_Comm comm, pcu_mpi_t* mpi) { + pcu_pmpi_init(comm, mpi); +} +void pcu_mpi_finalize(pcu_mpi_t* mpi) { + pcu_pmpi_finalize(mpi); } diff --git a/pcu/pcu_mpi.h b/pcu/pcu_mpi.h index 052920d04..970c8a4f2 100644 --- a/pcu/pcu_mpi.h +++ b/pcu/pcu_mpi.h @@ -13,6 +13,10 @@ #include "pcu_buffer.h" #include +#ifdef __cplusplus +extern "C" { +#endif + typedef struct { pcu_buffer buffer; @@ -23,21 +27,25 @@ typedef struct void pcu_make_message(pcu_message* m); void pcu_free_message(pcu_message* m); -typedef struct +struct pcu_mpi_struct { - int (*size)(void); - int (*rank)(void); - void (*send)(pcu_message* m, MPI_Comm comm); - bool (*done)(pcu_message* m); - bool (*receive)(pcu_message* m, MPI_Comm comm); -} pcu_mpi; - -void pcu_set_mpi(pcu_mpi* m); -pcu_mpi* pcu_get_mpi(void); -int pcu_mpi_size(void); -int pcu_mpi_rank(void); -void pcu_mpi_send(pcu_message* m, MPI_Comm comm); -bool pcu_mpi_done(pcu_message* m); -bool pcu_mpi_receive(pcu_message* m, MPI_Comm comm); - + MPI_Comm original_comm; + MPI_Comm user_comm; + MPI_Comm coll_comm; + int rank; + int size; +}; +typedef struct pcu_mpi_struct pcu_mpi_t; + +int pcu_mpi_size(const pcu_mpi_t*); +int pcu_mpi_rank(const pcu_mpi_t*); +void pcu_mpi_send(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +bool pcu_mpi_done(const pcu_mpi_t*, pcu_message* m); +bool pcu_mpi_receive(const pcu_mpi_t*, pcu_message* m, MPI_Comm comm); +void pcu_mpi_init(MPI_Comm comm, pcu_mpi_t* mpi); +void pcu_mpi_finalize(pcu_mpi_t* mpi); + +#ifdef __cplusplus +} +#endif #endif diff --git a/pcu/pcu_msg.c b/pcu/pcu_msg.c index 2b4705890..538310c30 100644 --- a/pcu/pcu_msg.c +++ b/pcu/pcu_msg.c @@ -8,7 +8,6 @@ *******************************************************************************/ #include "pcu_msg.h" -#include "pcu_pmpi.h" #include "noto_malloc.h" #include "reel.h" #include @@ -87,14 +86,14 @@ static void free_peers(pcu_aa_tree* t) pcu_make_aa(t); } -void pcu_msg_start(pcu_msg* m) +void pcu_msg_start(pcu_mpi_t* mpi, pcu_msg* m) { if (m->state != idle_state) reel_fail("PCU_Comm_Begin called at the wrong time"); /* this barrier ensures no one starts a new superstep while others are receiving in the past superstep. It is the only blocking call in the pcu_msg system. */ - pcu_barrier(&(m->coll)); + pcu_barrier(mpi, &(m->coll)); m->state = pack_state; } @@ -143,49 +142,49 @@ size_t pcu_msg_packed(pcu_msg* m, int id) return peer->message.buffer.size; } -static void send_peers(pcu_aa_tree t) +static void send_peers(pcu_mpi_t* mpi, pcu_aa_tree t) { if (pcu_aa_empty(t)) return; pcu_msg_peer* peer; peer = (pcu_msg_peer*)t; - pcu_mpi_send(&(peer->message),pcu_user_comm); - send_peers(t->left); - send_peers(t->right); + pcu_mpi_send(mpi, &(peer->message),mpi->user_comm); + send_peers(mpi, t->left); + send_peers(mpi, t->right); } -void pcu_msg_send(pcu_msg* m) +void pcu_msg_send(pcu_mpi_t* mpi, pcu_msg* m) { if (m->state != pack_state) reel_fail("PCU_Comm_Send called at the wrong time"); - send_peers(m->peers); + send_peers(mpi, m->peers); m->state = send_recv_state; } -static bool done_sending_peers(pcu_aa_tree t) +static bool done_sending_peers(pcu_mpi_t* mpi, pcu_aa_tree t) { if (pcu_aa_empty(t)) return true; pcu_msg_peer* peer; peer = (pcu_msg_peer*)t; - return pcu_mpi_done(&(peer->message)) - && done_sending_peers(t->left) - && done_sending_peers(t->right); + return pcu_mpi_done(mpi, &(peer->message)) + && done_sending_peers(mpi, t->left) + && done_sending_peers(mpi, t->right); } -static bool receive_global(pcu_msg* m) +static bool receive_global(pcu_mpi_t* mpi, pcu_msg* m) { m->received.peer = MPI_ANY_SOURCE; - while ( ! pcu_mpi_receive(&(m->received),pcu_user_comm)) + while ( ! pcu_mpi_receive(mpi, &(m->received),mpi->user_comm)) { if (m->state == send_recv_state) - if (done_sending_peers(m->peers)) + if (done_sending_peers(mpi, m->peers)) { - pcu_begin_barrier(&(m->coll)); + pcu_begin_barrier(mpi, &(m->coll)); m->state = recv_state; } if (m->state == recv_state) - if (pcu_barrier_done(&(m->coll))) + if (pcu_barrier_done(mpi, &(m->coll))) return false; } return true; @@ -197,14 +196,14 @@ static void free_comm(pcu_msg* m) pcu_free_message(&(m->received)); } -bool pcu_msg_receive(pcu_msg* m) +bool pcu_msg_receive(pcu_mpi_t* mpi, pcu_msg* m) { if ((m->state != send_recv_state)&& (m->state != recv_state)) reel_fail("PCU_Comm_Receive called at the wrong time"); if ( ! pcu_msg_unpacked(m)) reel_fail("PCU_Comm_Receive called before previous message unpacked"); - if (receive_global(m)) + if (receive_global(mpi, m)) { pcu_begin_buffer(&(m->received.buffer)); return true; diff --git a/pcu/pcu_msg.h b/pcu/pcu_msg.h index 48713e80a..ba7595ff6 100644 --- a/pcu/pcu_msg.h +++ b/pcu/pcu_msg.h @@ -14,6 +14,10 @@ #include "pcu_aa.h" #include "pcu_io.h" +#ifdef __cplusplus +extern "C" { +#endif + /* the PCU Messenger (pcu_msg for short) system implements a non-blocking Bulk Synchronous Parallel communication model it is based on PCU non-blocking Collectives and pcu_mpi, @@ -45,13 +49,13 @@ struct pcu_msg_struct typedef struct pcu_msg_struct pcu_msg; void pcu_make_msg(pcu_msg* m); -void pcu_msg_start(pcu_msg* b); +void pcu_msg_start(pcu_mpi_t*, pcu_msg* b); void* pcu_msg_pack(pcu_msg* m, int id, size_t size); #define PCU_MSG_PACK(m,id,o) \ memcpy(pcu_msg_pack(m,id,sizeof(o)),&(o),sizeof(o)) size_t pcu_msg_packed(pcu_msg* m, int id); -void pcu_msg_send(pcu_msg* m); -bool pcu_msg_receive(pcu_msg* m); +void pcu_msg_send(pcu_mpi_t *mpi, pcu_msg* m); +bool pcu_msg_receive(pcu_mpi_t* mpi, pcu_msg* m); void* pcu_msg_unpack(pcu_msg* m, size_t size); #define PCU_MSG_UNPACK(m,o) \ memcpy(&(o),pcu_msg_unpack(m,sizeof(o)),sizeof(o)) @@ -60,4 +64,7 @@ int pcu_msg_received_from(pcu_msg* m); size_t pcu_msg_received_size(pcu_msg* m); void pcu_free_msg(pcu_msg* m); +#ifdef __cplusplus +} +#endif #endif //PCU_MSG_H diff --git a/pcu/pcu_order.c b/pcu/pcu_order.c index 1099e2d80..d0989a056 100644 --- a/pcu/pcu_order.c +++ b/pcu/pcu_order.c @@ -102,10 +102,10 @@ static void fill(pcu_order o, pcu_aa_tree t) fill(o, t->right); } -static void prepare(pcu_order o, pcu_msg* t) +static void prepare(pcu_mpi_t* mpi, pcu_order o, pcu_msg* t) { struct message* m; - while (pcu_msg_receive(t)) { + while (pcu_msg_receive(mpi, t)) { m = take_message(t); pcu_aa_insert(&m->node, &o->tree, message_less); } @@ -117,10 +117,10 @@ static void prepare(pcu_order o, pcu_msg* t) o->ready = true; } -bool pcu_order_receive(pcu_order o, pcu_msg* m) +bool pcu_order_receive(pcu_mpi_t* mpi, pcu_order o, pcu_msg* m) { if (!o->ready) - prepare(o, m); + prepare(mpi, o, m); o->at++; if (o->at == o->count) { dtor_order(o); diff --git a/pcu/pcu_order.h b/pcu/pcu_order.h index b762a1752..1aa870edc 100644 --- a/pcu/pcu_order.h +++ b/pcu/pcu_order.h @@ -13,14 +13,20 @@ #include #include "pcu_msg.h" +#ifdef __cplusplus +extern "C" { +#endif typedef struct pcu_order_struct* pcu_order; pcu_order pcu_order_new(void); void pcu_order_free(pcu_order o); -bool pcu_order_receive(pcu_order o, pcu_msg* m); +bool pcu_order_receive(pcu_mpi_t*, pcu_order o, pcu_msg* m); void* pcu_order_unpack(pcu_order o, size_t size); bool pcu_order_unpacked(pcu_order o); int pcu_order_received_from(pcu_order o); size_t pcu_order_received_size(pcu_order o); +#ifdef __cplusplus +} +#endif #endif diff --git a/pcu/pcu_pmpi.c b/pcu/pcu_pmpi.c index 814828a7e..1db376d2c 100644 --- a/pcu/pcu_pmpi.c +++ b/pcu/pcu_pmpi.c @@ -13,52 +13,43 @@ #include #include -static int global_size; -static int global_rank; +void pcu_pmpi_send2(const pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm); +bool pcu_pmpi_receive2(const pcu_mpi_t*, pcu_message* m, int tag, MPI_Comm comm); -MPI_Comm original_comm; -MPI_Comm pcu_user_comm; -MPI_Comm pcu_coll_comm; - -pcu_mpi pcu_pmpi = -{ .size = pcu_pmpi_size, - .rank = pcu_pmpi_rank, - .send = pcu_pmpi_send, - .done = pcu_pmpi_done, - .receive = pcu_pmpi_receive }; - -void pcu_pmpi_init(MPI_Comm comm) +void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t* self) { - original_comm = comm; - MPI_Comm_dup(comm,&pcu_user_comm); - MPI_Comm_dup(comm,&pcu_coll_comm); - MPI_Comm_size(comm,&global_size); - MPI_Comm_rank(comm,&global_rank); + self->original_comm = comm; + MPI_Comm_dup(comm,&(self->user_comm)); + MPI_Comm_dup(comm,&(self->coll_comm)); + MPI_Comm_size(comm,&(self->size)); + MPI_Comm_rank(comm,&(self->rank)); } -void pcu_pmpi_finalize(void) +void pcu_pmpi_finalize(pcu_mpi_t* self) { - MPI_Comm_free(&pcu_user_comm); - MPI_Comm_free(&pcu_coll_comm); + MPI_Comm_free(&(self->user_comm)); + MPI_Comm_free(&(self->coll_comm)); } -int pcu_pmpi_size(void) +int pcu_pmpi_size(const pcu_mpi_t* self) { - return global_size; + return self->size; } -int pcu_pmpi_rank(void) +int pcu_pmpi_rank(const pcu_mpi_t* self) { - return global_rank; + return self->rank; } -void pcu_pmpi_send(pcu_message* m, MPI_Comm comm) +void pcu_pmpi_send(const pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { - pcu_pmpi_send2(m,0,comm); + pcu_pmpi_send2(self, m,0,comm); } -void pcu_pmpi_send2(pcu_message* m, int tag, MPI_Comm comm) +void pcu_pmpi_send2(const pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) { + // silence warning + (void)self; if( m->buffer.size > (size_t)INT_MAX ) { fprintf(stderr, "ERROR PCU message size exceeds INT_MAX... exiting\n"); abort(); @@ -73,20 +64,26 @@ void pcu_pmpi_send2(pcu_message* m, int tag, MPI_Comm comm) &(m->request)); } -bool pcu_pmpi_done(pcu_message* m) +bool pcu_pmpi_done(const pcu_mpi_t* self, pcu_message* m) { + // silence warning + (void)self; int flag; MPI_Test(&(m->request),&flag,MPI_STATUS_IGNORE); return flag; } -bool pcu_pmpi_receive(pcu_message* m, MPI_Comm comm) +bool pcu_pmpi_receive(const pcu_mpi_t* self, pcu_message* m, MPI_Comm comm) { - return pcu_pmpi_receive2(m,0,comm); + // silence warning + (void)self; + return pcu_pmpi_receive2(self, m,0,comm); } -bool pcu_pmpi_receive2(pcu_message* m, int tag, MPI_Comm comm) +bool pcu_pmpi_receive2(const pcu_mpi_t* self, pcu_message* m, int tag, MPI_Comm comm) { + // silence warning + (void)self; MPI_Status status; int flag; MPI_Iprobe(m->peer,tag,comm,&flag,&status); @@ -106,15 +103,3 @@ bool pcu_pmpi_receive2(pcu_message* m, int tag, MPI_Comm comm) MPI_STATUS_IGNORE); return true; } - -void pcu_pmpi_switch(MPI_Comm new_comm) -{ - pcu_pmpi_finalize(); - pcu_pmpi_init(new_comm); -} - -MPI_Comm pcu_pmpi_comm(void) -{ - return original_comm; -} - diff --git a/pcu/pcu_pmpi.h b/pcu/pcu_pmpi.h index c6707d23e..7505aa7a4 100644 --- a/pcu/pcu_pmpi.h +++ b/pcu/pcu_pmpi.h @@ -1,8 +1,8 @@ -/****************************************************************************** +/****************************************************************************** - Copyright 2011 Scientific Computation Research Center, + Copyright 2011 Scientific Computation Research Center, Rensselaer Polytechnic Institute. All rights reserved. - + This work is open source software, licensed under the terms of the BSD license as described in the LICENSE file in the top-level directory. @@ -13,23 +13,18 @@ #include "pcu_mpi.h" #include - -void pcu_pmpi_init(MPI_Comm comm); -void pcu_pmpi_finalize(void); -int pcu_pmpi_size(void); -int pcu_pmpi_rank(void); -void pcu_pmpi_send(pcu_message* m, MPI_Comm comm); -bool pcu_pmpi_receive(pcu_message* m, MPI_Comm comm); -void pcu_pmpi_send2(pcu_message* m, int tag, MPI_Comm comm); -bool pcu_pmpi_receive2(pcu_message* m, int tag, MPI_Comm comm); -bool pcu_pmpi_done(pcu_message* m); - -void pcu_pmpi_switch(MPI_Comm new_comm); -MPI_Comm pcu_pmpi_comm(void); - -extern pcu_mpi pcu_pmpi; - -extern MPI_Comm pcu_user_comm; -extern MPI_Comm pcu_coll_comm; - +#ifdef __cplusplus +extern "C" { +#endif +void pcu_pmpi_init(MPI_Comm comm, pcu_mpi_t *mpi); +void pcu_pmpi_finalize(pcu_mpi_t *m); +int pcu_pmpi_size(const pcu_mpi_t *self); +int pcu_pmpi_rank(const pcu_mpi_t *self); +void pcu_pmpi_send(const pcu_mpi_t *, pcu_message *m, MPI_Comm comm); +bool pcu_pmpi_receive(const pcu_mpi_t *, pcu_message *m, MPI_Comm comm); +bool pcu_pmpi_done(const pcu_mpi_t *, pcu_message *m); + +#ifdef __cplusplus +} +#endif #endif diff --git a/pcu/pcu_util.h b/pcu/pcu_util.h index e61394e51..9a1d1fa7d 100644 --- a/pcu/pcu_util.h +++ b/pcu/pcu_util.h @@ -22,25 +22,53 @@ void PCU_Assert_Fail(const char* msg) __attribute__ ((noreturn)); } /* extern "C" */ #endif -#define PCU_ALWAYS_ASSERT(cond) \ - do { \ - if (! (cond)) { \ - char omsg[2048]; \ - snprintf(omsg, 2048, "%s failed at %s + %d \n", \ - #cond, __FILE__, __LINE__); \ - PCU_Assert_Fail(omsg); \ - } \ - } while (0) - -#define PCU_ALWAYS_ASSERT_VERBOSE(cond, msg) \ - do { \ - if (! (cond)) { \ - char omsg[2048]; \ - snprintf(omsg, 2048, "%s failed at %s + %d \n %s", \ - #cond, __FILE__, __LINE__, msg); \ - PCU_Assert_Fail(omsg); \ - } \ - } while(0) +#define PCU_DO_PRAGMA_(x) _Pragma (#x) +#define PCU_DO_PRAGMA(x) PCU_DO_PRAGMA_(x) + +#if defined(__GNUC__) + +#define PCU_IGNORE_DIAGNOSTIC_START(x) \ + PCU_DO_PRAGMA(GCC diagnostic push) \ + PCU_DO_PRAGMA(GCC diagnostic ignored #x) +#define PCU_IGNORE_DIAGNOSTIC_END \ + PCU_DO_PRAGMA(GCC diagnostic pop) + +#elif defined(__clang__) + +#define PCU_IGNORE_DIAGNOSTIC_START(x) \ + PCU_DO_PRAGMA(clang diagnostic push) \ + PCU_DO_PRAGMA(clang diagnostic ignored #x) +#define PCU_IGNORE_DIAGNOSTIC_END \ + PCU_DO_PRAGMA(clang diagnostic pop) + +#else +#define PCU_IGNORE_DIAGNOSTIC_START(x) +#define PCU_IGNORE_DIAGNOSTIC_END +#endif + + +#define PCU_ALWAYS_ASSERT(cond) \ + do { \ + PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations) \ + if (! (cond)) { \ + char omsg[2048]; \ + snprintf(omsg, 2048, "%s failed at %s + %d \n", \ + #cond, __FILE__, __LINE__); \ + PCU_Assert_Fail(omsg); \ + } \ + PCU_IGNORE_DIAGNOSTIC_END \ + } while (0) +#define PCU_ALWAYS_ASSERT_VERBOSE(cond, msg) \ + do { \ + PCU_IGNORE_DIAGNOSTIC_START(-Wdeprecated-declarations) \ + if (! (cond)) { \ + char omsg[2048]; \ + snprintf(omsg, 2048, "%s failed at %s + %d \n %s", \ + #cond, __FILE__, __LINE__, msg); \ + PCU_Assert_Fail(omsg); \ + } \ + PCU_IGNORE_DIAGNOSTIC_END \ + } while(0) #ifdef NDEBUG #define PCU_DEBUG_ASSERT(cond) diff --git a/pcu/pkg_tribits.cmake b/pcu/pkg_tribits.cmake index 0c43fc64a..86278a4a2 100644 --- a/pcu/pkg_tribits.cmake +++ b/pcu/pkg_tribits.cmake @@ -33,27 +33,29 @@ include_directories("${PROJECT_BINARY_DIR}") #Sources & Headers set(SOURCES - pcu.c - pcu_aa.c - pcu_coll.c - pcu_io.c - pcu_buffer.c - pcu_mpi.c - pcu_msg.c - pcu_order.c - pcu_pmpi.c - pcu_util.c - noto/noto_malloc.c - reel/reel.c -) + pcu_c.cc + pcu_aa.c + pcu_coll.c + pcu_io.c + pcu_buffer.c + pcu_mem.c + pcu_mpi.c + pcu_msg.c + pcu_order.c + pcu_pmpi.c + pcu_util.c + noto/noto_malloc.c + reel/reel.c + PCU.cc) set(HEADERS - PCU.h - pcu_io.h - pcu_util.h - noto/noto_malloc.h - reel/reel.h) - + PCU_C.h + pcu_io.h + pcu_util.h + reel/reel.h + pcu_defines.h + PCU.h + ) tribits_add_library( pcu HEADERS ${HEADERS} diff --git a/phasta/adaptLvlSet_loop.cc b/phasta/adaptLvlSet_loop.cc index df1fc3727..c5ac347a6 100644 --- a/phasta/adaptLvlSet_loop.cc +++ b/phasta/adaptLvlSet_loop.cc @@ -10,8 +10,6 @@ generates PHASTA files containing the mesh and field information. */ - -#include #include #include #include @@ -39,8 +37,9 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); - PCU_Protect(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::Protect(); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); gmi_sim_start(); @@ -49,28 +48,28 @@ int main(int argc, char** argv) { gmi_register_mesh(); gmi_model* g = NULL; apf::Mesh2* m = NULL; - GRStream* grs = makeGRStream(); + GRStream* grs = makeGRStream(&PCUObj); ph::Input ctrl; - ctrl.load("adaptLvlSet.inp"); + ctrl.load("adaptLvlSet.inp", &PCUObj); //preprocess (define bubbles) - chef::cook(g,m,ctrl,grs); - RStream* rs = makeRStream(); - attachRStream(grs,rs); + chef::cook(g,m,ctrl,grs,&PCUObj); + RStream* rs = makeRStream(&PCUObj); + attachRStream(grs,rs,&PCUObj); reconfigureChef(ctrl); // attach the solution field from stream, adapt to the level set, // and then preprocess (redefines bubbles) - chef::cook(g,m,ctrl,rs,grs); - attachRStream(grs,rs); + chef::cook(g,m,ctrl,rs,grs,&PCUObj); + attachRStream(grs,rs,&PCUObj); // attach a solution field from stream, adapt to the level set, // and then preprocess (redefines bubbles) - chef::cook(g,m,ctrl,rs); - destroyGRStream(grs); - destroyRStream(rs); + chef::cook(g,m,ctrl,rs,&PCUObj); + destroyGRStream(grs,&PCUObj); + destroyRStream(rs,&PCUObj); freeMesh(m); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); Sim_unregisterAllKeys(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/chef.cc b/phasta/chef.cc index c946cab2d..1fd034d8c 100644 --- a/phasta/chef.cc +++ b/phasta/chef.cc @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -31,10 +30,11 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); - PCU_Protect(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::Protect(); lion_set_verbosity(1); - if( !PCU_Comm_Self() ) { + if( !PCUObj.Self() ) { lion_oprint(1,"PUMI Git hash %s\n", pumi_version()); lion_oprint(1,"PUMI version %s Git hash %s\n", pumi_version(), pumi_git_sha()); } @@ -55,8 +55,8 @@ int main(int argc, char** argv) std::string inputPath = "adapt.inp"; if(argc==2) inputPath = argv[1]; ph::Input in; - in.load(inputPath.c_str()); - chef::cook(g,m,in); + in.load(inputPath.c_str(), &PCUObj); + chef::cook(g,m,in,&PCUObj); freeMesh(m); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); @@ -68,7 +68,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/chef.h b/phasta/chef.h index c58a6ab3e..d60c0bb32 100644 --- a/phasta/chef.h +++ b/phasta/chef.h @@ -13,18 +13,18 @@ struct RStream; struct GRStream; namespace chef { /** @brief read and write to and from files */ - void cook(gmi_model*& g, apf::Mesh2*& m); + void cook(gmi_model*& g, apf::Mesh2*& m, pcu::PCU *PCUObj); /** @brief read and write to and from files and load control info*/ - void cook(gmi_model*& g, apf::Mesh2*& m, ph::Input& ctrl); + void cook(gmi_model*& g, apf::Mesh2*& m, ph::Input& ctrl, pcu::PCU *PCUObj); /** @brief read from stream and write to files */ void cook(gmi_model*& g, apf::Mesh2*& m, - ph::Input& ctrl, RStream* in); + ph::Input& ctrl, RStream* in, pcu::PCU *PCUObj); /** @brief read from files and write to stream */ void cook(gmi_model*& g, apf::Mesh2*& m, - ph::Input& ctrl, GRStream* out); + ph::Input& ctrl, GRStream* out, pcu::PCU *PCUObj); /** @brief read and write to and from streams */ void cook(gmi_model*& g, apf::Mesh2*& m, - ph::Input& ctrl, RStream* in, GRStream* out); + ph::Input& ctrl, RStream* in, GRStream* out, pcu::PCU *PCUObj); /** @brief extract a field from a packed field */ apf::Field* extractField(apf::Mesh* m, const char* packedFieldname, diff --git a/phasta/chefStream.cc b/phasta/chefStream.cc index 79ff422f7..37f79e8c4 100644 --- a/phasta/chefStream.cc +++ b/phasta/chefStream.cc @@ -10,8 +10,6 @@ generates PHASTA files containing the mesh and field information. */ - -#include #include #include #include @@ -35,8 +33,9 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); - PCU_Protect(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pcu::Protect(); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); gmi_sim_start(); @@ -45,21 +44,21 @@ int main(int argc, char** argv) { gmi_register_mesh(); gmi_model* g = NULL; apf::Mesh2* m = NULL; - GRStream* grs = makeGRStream(); + GRStream* grs = makeGRStream(&PCUObj); ph::Input ctrl; - ctrl.load("adapt.inp"); - chef::cook(g,m,ctrl,grs); - RStream* rs = makeRStream(); - attachRStream(grs,rs); + ctrl.load("adapt.inp", &PCUObj); + chef::cook(g,m,ctrl,grs,&PCUObj); + RStream* rs = makeRStream(&PCUObj); + attachRStream(grs,rs,&PCUObj); reconfigureChef(ctrl); - chef::cook(g,m,ctrl,rs); - destroyGRStream(grs); - destroyRStream(rs); + chef::cook(g,m,ctrl,rs,&PCUObj); + destroyGRStream(grs,&PCUObj); + destroyRStream(rs,&PCUObj); freeMesh(m); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); Sim_unregisterAllKeys(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/condense.cc b/phasta/condense.cc index e703059fd..6e74a892b 100644 --- a/phasta/condense.cc +++ b/phasta/condense.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -14,7 +13,7 @@ #include namespace { - static FILE* openfile_read(ph::Input&, const char* path) { + static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU*) { return fopen(path, "r"); } @@ -27,24 +26,25 @@ namespace { } }; - void checkInputs(int argc, char** argv) { + void checkInputs(int argc, char** argv, pcu::PCU *pcu_obj) { if ( argc != 3 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj->Self() ) lion_oprint(1,"Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } - if ( !PCU_Comm_Self() ) + if ( !pcu_obj->Self() ) lion_oprint(1,"Input control file %s reduction factor %s\n", argv[1], argv[2]); } } int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); - PCU_Protect(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::Protect(); lion_set_verbosity(1); - checkInputs(argc,argv); + checkInputs(argc,argv,&pcu_obj); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); gmi_sim_start(); @@ -53,17 +53,17 @@ int main(int argc, char** argv) { gmi_register_mesh(); GroupCode code; ph::Input in; - in.load(argv[1]); + in.load(argv[1], &pcu_obj); in.openfile_read = openfile_read; - code.mesh = apf::loadMdsMesh(in.modelFileName.c_str(), in.meshFileName.c_str()); + code.mesh = apf::loadMdsMesh(in.modelFileName.c_str(), in.meshFileName.c_str(), &pcu_obj); chef::readAndAttachFields(in,code.mesh); - apf::Unmodulo outMap(PCU_Comm_Self(), PCU_Comm_Peers()); + apf::Unmodulo outMap(code.mesh->getPCU()->Self(), code.mesh->getPCU()->Peers()); code.ctrl = in; Parma_ShrinkPartition(code.mesh, atoi(argv[2]), code); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); Sim_unregisterAllKeys(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/cut_interface.cc b/phasta/cut_interface.cc index d600211e5..bf4a9e131 100644 --- a/phasta/cut_interface.cc +++ b/phasta/cut_interface.cc @@ -1,7 +1,6 @@ #include "ph.h" #include "phInterfaceCutter.h" #include "phAttrib.h" -#include #include #ifdef HAVE_SIMMETRIX #include @@ -32,8 +31,9 @@ int main(int argc, char** argv) lion_eprint(1," to take combined model and attributes file (by simTranslate)\n"); return 0; } - PCU_Comm_Init(); - PCU_ALWAYS_ASSERT(PCU_Comm_Peers() == 1); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + PCU_ALWAYS_ASSERT(PCUObj.Peers() == 1); #ifdef HAVE_SIMMETRIX SimModel_start(); Sim_readLicenseFile(0); @@ -60,7 +60,7 @@ int main(int argc, char** argv) gm = gmi_sim_load(modelfile, attribfile); ph::BCs bcs; ph::getSimmetrixAttributes(gm, bcs); - apf::Mesh2* m = ph::loadMesh(gm, meshfile); + apf::Mesh2* m = ph::loadMesh(gm, meshfile, &PCUObj); m->verify(); #ifdef HAVE_SIMMETRIX if (ph::mesh_has_ext(meshfile, "sms")) @@ -81,6 +81,6 @@ int main(int argc, char** argv) SimModel_stop(); Sim_unregisterAllKeys(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/migrate_interface.cc b/phasta/migrate_interface.cc index 804e9098c..e45bc3d67 100644 --- a/phasta/migrate_interface.cc +++ b/phasta/migrate_interface.cc @@ -2,7 +2,6 @@ #include "phBC.h" #include "phInterfaceCutter.h" #include -#include #include #include #include @@ -31,10 +30,10 @@ void freeMesh(apf::Mesh* m) apf::destroyMesh(m); } -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *pcu_obj) { if (argc < 4 || argc > 5) { - if ( !PCU_Comm_Self() ) { + if ( !pcu_obj->Self() ) { lion_eprint(1,"Usage: %s \n", argv[0]); lion_eprint(1," to take model and attributes in separate files\n"); lion_eprint(1,"Usage: %s \n", argv[0]); @@ -58,7 +57,8 @@ void getConfig(int argc, char** argv) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -68,11 +68,11 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv); + getConfig(argc,argv,&pcu_obj); gmi_model* gm; gm = gmi_sim_load(modelFile, attribFile); - apf::Mesh2* m = apf::loadMdsMesh(gm, inMesh); + apf::Mesh2* m = apf::loadMdsMesh(gm, inMesh, &pcu_obj); m->verify(); ph::BCs bcs; ph::getSimmetrixAttributes(gm, bcs); @@ -89,6 +89,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/ph.cc b/phasta/ph.cc index 17b404120..417122b61 100644 --- a/phasta/ph.cc +++ b/phasta/ph.cc @@ -1,4 +1,3 @@ -#include #include #include "ph.h" @@ -63,14 +62,14 @@ void goToParentDir() { my_chdir(".."); } -void goToStepDir(int step, bool all_mkdir) +void goToStepDir(int step, pcu::PCU *PCUObj, bool all_mkdir) { std::stringstream ss; ss << step; std::string s = ss.str(); - if (all_mkdir || !PCU_Comm_Self()) + if (all_mkdir || !PCUObj->Self()) my_mkdir(s.c_str()); - PCU_Barrier(); + PCUObj->Barrier(); my_chdir(s.c_str()); } @@ -78,22 +77,22 @@ enum { DIR_FANOUT = 2048 }; -std::string setupOutputDir(bool all_mkdir) +std::string setupOutputDir(pcu::PCU *PCUObj, bool all_mkdir) { std::stringstream ss; - ss << PCU_Comm_Peers() << "-procs_case/"; + ss << PCUObj->Peers() << "-procs_case/"; std::string s = ss.str(); - if (all_mkdir || !PCU_Comm_Self()) + if (all_mkdir || !PCUObj->Self()) my_mkdir(s.c_str()); - PCU_Barrier(); + PCUObj->Barrier(); return s; } -void setupOutputSubdir(std::string& path, bool all_mkdir) +void setupOutputSubdir(std::string& path, pcu::PCU *PCUObj, bool all_mkdir) { - if (PCU_Comm_Peers() <= DIR_FANOUT) + if (PCUObj->Peers() <= DIR_FANOUT) return; - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); int subSelf = self % DIR_FANOUT; int subGroup = self / DIR_FANOUT; std::stringstream ss; @@ -101,14 +100,14 @@ void setupOutputSubdir(std::string& path, bool all_mkdir) path = ss.str(); if (all_mkdir || !subSelf) my_mkdir(path.c_str()); - PCU_Barrier(); + PCUObj->Barrier(); } -void setupInputSubdir(std::string& path) +void setupInputSubdir(std::string& path, pcu::PCU *PCUObj) { - if (PCU_Comm_Peers() <= DIR_FANOUT) + if (PCUObj->Peers() <= DIR_FANOUT) return; - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); int subGroup = self / DIR_FANOUT; std::string newpath; std::stringstream ss; @@ -127,16 +126,16 @@ void setupInputSubdir(std::string& path) // lion_oprint(1,"Rank: %d - Path in setupInputSubdir: %s\n", self, ss.str().c_str()); path = ss.str(); - PCU_Barrier(); + PCUObj->Barrier(); } -void writeAuxiliaryFiles(std::string path, int timestep_or_dat) +void writeAuxiliaryFiles(std::string path, int timestep_or_dat, pcu::PCU *PCUObj) { std::string numpePath = path; numpePath += "numpe.in"; std::ofstream numpe(numpePath.c_str()); PCU_ALWAYS_ASSERT(numpe.is_open()); - numpe << PCU_Comm_Peers() << '\n'; + numpe << PCUObj->Peers() << '\n'; numpe.close(); std::string numstartPath = path; numstartPath += "numstart.dat"; @@ -157,7 +156,7 @@ bool mesh_has_ext(const char* filename, const char* ext) } } -apf::Mesh2* loadMesh(gmi_model*& g, const char* meshfile) { +apf::Mesh2* loadMesh(gmi_model*& g, const char* meshfile, pcu::PCU *PCUObj) { apf::Mesh2* mesh; #ifdef HAVE_SIMMETRIX /* if it is a simmetrix mesh */ @@ -167,14 +166,14 @@ apf::Mesh2* loadMesh(gmi_model*& g, const char* meshfile) { pGModel simModel = gmi_export_sim(g); pParMesh sim_mesh = PM_load(meshfile, simModel, progress); - mesh = apf::createMesh(sim_mesh); + mesh = apf::createMesh(sim_mesh, PCUObj); Progress_delete(progress); } else #endif /* if it is a SCOREC mesh */ { - mesh = apf::loadMdsMesh(g, meshfile); + mesh = apf::loadMdsMesh(g, meshfile, PCUObj); } return mesh; } diff --git a/phasta/ph.h b/phasta/ph.h index c3ab09508..0b869bb1e 100644 --- a/phasta/ph.h +++ b/phasta/ph.h @@ -9,14 +9,14 @@ namespace ph { void fail(const char* format, ...) __attribute__((noreturn,format(printf,1,2))); -void goToStepDir(int step, bool all_mkdir=false); +void goToStepDir(int step, pcu::PCU *PCUObj, bool all_mkdir=false); void goToParentDir(); -std::string setupOutputDir(bool all_mkdir=false); -void setupInputSubdir(std::string& path); -void setupOutputSubdir(std::string& path, bool all_mkdir=false); -void writeAuxiliaryFiles(std::string path, int timestep_or_dat); +std::string setupOutputDir(pcu::PCU *PCUObj, bool all_mkdir=false); +void setupInputSubdir(std::string& path, pcu::PCU *PCUObj); +void setupOutputSubdir(std::string& path, pcu::PCU *PCUObj, bool all_mkdir=false); +void writeAuxiliaryFiles(std::string path, int timestep_or_dat, pcu::PCU *PCUObj); bool mesh_has_ext(const char* filename, const char* ext); -apf::Mesh2* loadMesh(gmi_model*& g, const char* meshfile); +apf::Mesh2* loadMesh(gmi_model*& g, const char* meshfile, pcu::PCU *PCUObj); } diff --git a/phasta/phAdapt.cc b/phasta/phAdapt.cc index 9ae8f783a..a720386c8 100644 --- a/phasta/phAdapt.cc +++ b/phasta/phAdapt.cc @@ -2,7 +2,6 @@ #include "chef.h" #include "ph.h" #include -#include #include #include #include @@ -14,7 +13,7 @@ namespace ph { void setupBalance(const char* key, std::string& method, - bool& parmaBal, bool& zoltanBal, bool& zoltanRibBal) { + bool& parmaBal, bool& zoltanBal, bool& zoltanRibBal, pcu::PCU *PCUObj) { if ( method == "parma" ) { parmaBal = true; zoltanBal = false; @@ -32,15 +31,15 @@ void setupBalance(const char* key, std::string& method, zoltanBal = false; zoltanRibBal = false; } else { - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) lion_eprint(1, "warning: ignoring unknown value of %s = %s\n", key, method.c_str()); } } -void setupMatching(ma::Input& in) { - if (!PCU_Comm_Self()) +void setupMatching(ma::Input& in, pcu::PCU *PCUObj) { + if (!PCUObj->Self()) lion_oprint(1,"Matched mesh: disabling" " snapping, and shape correction,\n"); in.shouldSnap = false; @@ -66,13 +65,13 @@ struct AdaptCallback : public Parma_GroupCode //override with user inputs if specified setupBalance("preAdaptBalanceMethod", in->preAdaptBalanceMethod, ma_in->shouldRunPreParma, ma_in->shouldRunPreZoltan, - ma_in->shouldRunPreZoltanRib); + ma_in->shouldRunPreZoltanRib, mesh->getPCU()); bool ignored; setupBalance("midAdaptBalanceMethod", in->midAdaptBalanceMethod, - ma_in->shouldRunMidParma, ma_in->shouldRunMidZoltan, ignored); + ma_in->shouldRunMidParma, ma_in->shouldRunMidZoltan, ignored, mesh->getPCU()); setupBalance("postAdaptBalanceMethod", in->postAdaptBalanceMethod, ma_in->shouldRunPostParma, ma_in->shouldRunPostZoltan, - ma_in->shouldRunPostZoltanRib); + ma_in->shouldRunPostZoltanRib, mesh->getPCU()); ma_in->shouldTransferParametric = in->transferParametric; ma_in->shouldSnap = in->snap; ma_in->maximumIterations = in->maxAdaptIterations; @@ -88,33 +87,33 @@ struct AdaptCallback : public Parma_GroupCode ma_in->shouldRunPreZoltan = true; } if (mesh->hasMatching()) - ph::setupMatching(*ma_in); + ph::setupMatching(*ma_in, mesh->getPCU()); ma::adapt(ma_in); } }; static double getAveragePartDensity(apf::Mesh* m) { double nElements = m->count(m->getDimension()); - nElements = PCU_Add_Double(nElements); - return nElements / PCU_Comm_Peers(); + nElements = m->getPCU()->Add(nElements); + return nElements / m->getPCU()->Peers(); } static int getShrinkFactor(apf::Mesh* m, double minPartDensity) { double partDensity = getAveragePartDensity(m); int factor = 1; while (partDensity < minPartDensity) { - if (factor >= PCU_Comm_Peers()) + if (factor >= m->getPCU()->Peers()) break; factor *= 2; partDensity *= 2; } - PCU_ALWAYS_ASSERT(PCU_Comm_Peers() % factor == 0); + PCU_ALWAYS_ASSERT(m->getPCU()->Peers() % factor == 0); return factor; } -static void warnAboutShrinking(int factor) { - int nprocs = PCU_Comm_Peers() / factor; - if (!PCU_Comm_Self()) { +static void warnAboutShrinking(int factor, pcu::PCU *PCUObj) { + int nprocs = PCUObj->Peers() / factor; + if (!PCUObj->Self()) { lion_eprint(1,"sensing mesh is spread too thin: " "adapting with %d procs\n", nprocs); } @@ -123,12 +122,12 @@ static void warnAboutShrinking(int factor) { void adaptShrunken(apf::Mesh2* m, double minPartDensity, Parma_GroupCode& callback) { int factor = getShrinkFactor(m, minPartDensity); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_eprint(1,"adaptShrunken limit set to %f factor computed as %d\n", minPartDensity, factor); if (factor == 1) { callback.run(0); } else { - warnAboutShrinking(factor); + warnAboutShrinking(factor, m->getPCU()); Parma_ShrinkPartition(m, factor, callback); } } @@ -159,7 +158,7 @@ void tetrahedronize(Input& in, apf::Mesh2* m) ma::Input* ma_in = ma::makeAdvanced(ma::configureIdentity(m)); ph::setupBalance("preAdaptBalanceMethod", in.preAdaptBalanceMethod, ma_in->shouldRunPreParma, ma_in->shouldRunPreZoltan, - ma_in->shouldRunPreZoltanRib); + ma_in->shouldRunPreZoltanRib, m->getPCU()); ma_in->shouldTurnLayerToTets = true; ma::adapt(ma_in); m->verify(); @@ -215,7 +214,7 @@ namespace chef { ma::Input* ma_in = ma::makeAdvanced(ma::configureMatching(m, in.recursiveUR)); ph::setupBalance("preAdaptBalanceMethod", in.preAdaptBalanceMethod, ma_in->shouldRunPreParma, ma_in->shouldRunPreZoltan, - ma_in->shouldRunPreZoltanRib); + ma_in->shouldRunPreZoltanRib, m->getPCU()); // get the level set apf::Field* soln = m->findField("solution"); PCU_ALWAYS_ASSERT(soln); @@ -247,7 +246,7 @@ namespace chef { ma::Input* ma_in = ma::makeAdvanced(ma::configureMatching(m, in.recursiveUR)); ph::setupBalance("preAdaptBalanceMethod", in.preAdaptBalanceMethod, ma_in->shouldRunPreParma, ma_in->shouldRunPreZoltan, - ma_in->shouldRunPreZoltanRib); + ma_in->shouldRunPreZoltanRib, m->getPCU()); ma_in->shouldRefineLayer = true; ma_in->splitAllLayerEdges = in.splitAllLayerEdges; if (in.snap) { diff --git a/phasta/phAxisymmetry.cc b/phasta/phAxisymmetry.cc index 5aebe1dc8..132baa001 100644 --- a/phasta/phAxisymmetry.cc +++ b/phasta/phAxisymmetry.cc @@ -1,6 +1,5 @@ #include "phAxisymmetry.h" #include -#include #include #include @@ -111,7 +110,7 @@ apf::MeshTag* tagAngles(apf::Mesh* m, BCs& bcs, apf::MatchedSharing* ms) { apf::MeshTag* tag = m->createDoubleTag("ph_angle", 1); gmi_model* gm = m->getModel(); - PCU_Comm_Begin(); + m->getPCU()->Begin(); apf::MeshIterator* it = m->begin(0); apf::MeshEntity* v; while ((v = m->iterate(it))) { @@ -126,18 +125,18 @@ apf::MeshTag* tagAngles(apf::Mesh* m, BCs& bcs, apf::MatchedSharing* ms) mdim = m->getModelType(me); mtag = m->getModelTag(me); APF_ITERATE(apf::Matches, matches, mit) { - PCU_COMM_PACK(mit->peer, mit->entity); - PCU_COMM_PACK(mit->peer, mdim); - PCU_COMM_PACK(mit->peer, mtag); + m->getPCU()->Pack(mit->peer, mit->entity); + m->getPCU()->Pack(mit->peer, mdim); + m->getPCU()->Pack(mit->peer, mtag); } } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { - PCU_COMM_UNPACK(v); + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { + m->getPCU()->Unpack(v); int mdim, mtag; - PCU_COMM_UNPACK(mdim); - PCU_COMM_UNPACK(mtag); + m->getPCU()->Unpack(mdim); + m->getPCU()->Unpack(mtag); gmi_ent* oge = gmi_find(gm, mdim, mtag); gmi_ent* ge = (gmi_ent*) m->toModel(v); double angle; diff --git a/phasta/phBC.cc b/phasta/phBC.cc index 844fba905..f50836170 100644 --- a/phasta/phBC.cc +++ b/phasta/phBC.cc @@ -1,4 +1,3 @@ -#include #include #include "phBC.h" #ifdef HAVE_SIMMETRIX @@ -123,9 +122,9 @@ void readBCs(gmi_model* m, const char* attFile, bool axisymmetry, BCs& bcs) attachAllAngleBCs(m, bcs); } -void loadModelAndBCs(ph::Input& in, gmi_model*& m, BCs& bcs) +void loadModelAndBCs(ph::Input& in, gmi_model*& m, BCs& bcs, pcu::PCU *PCUObj) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); const char* modelfile = in.modelFileName.c_str(); const char* attribfile = in.attributeFileName.c_str(); /* loading the model */ @@ -148,8 +147,8 @@ void loadModelAndBCs(ph::Input& in, gmi_model*& m, BCs& bcs) #endif /* load attributes */ readBCs(m, attribfile, in.axisymmetry, bcs); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!PCUObj->Self()) lion_oprint(1,"\"%s\" and \"%s\" loaded in %f seconds\n", modelfile, attribfile, t1 - t0); } diff --git a/phasta/phBC.h b/phasta/phBC.h index 36e97c2d6..fe88fdf1b 100644 --- a/phasta/phBC.h +++ b/phasta/phBC.h @@ -4,7 +4,6 @@ #include #include #include - #include #include #include "phInput.h" @@ -128,7 +127,7 @@ struct BCs }; void readBCs(gmi_model* m, const char* attFile, bool axisymmetry, BCs& bcs); -void loadModelAndBCs(ph::Input& in, gmi_model*& m, BCs& bcs); +void loadModelAndBCs(ph::Input& in, gmi_model*& m, BCs& bcs, pcu::PCU *PCUObj); bool applyNaturalBCs(gmi_model* gm, gmi_ent* ge, BCs& appliedBCs, apf::Vector3 const& x, double* values, int* bits); diff --git a/phasta/phBlock.cc b/phasta/phBlock.cc index 573532e6d..277472d77 100644 --- a/phasta/phBlock.cc +++ b/phasta/phBlock.cc @@ -2,7 +2,6 @@ #include "phBC.h" #include #include -#include #include namespace ph { @@ -202,7 +201,7 @@ void getInterfaceBlocks(apf::Mesh* m, BCs& bcs, BlocksInterface& b) if (dgCopies.getSize() != 1) continue; apf::MeshEntity* e0 = m->getUpward(face, 0); - PCU_ALWAYS_ASSERT(dgCopies[0].peer == PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(dgCopies[0].peer == m->getPCU()->Self()); apf::MeshEntity* e1 = m->getUpward(dgCopies[0].entity, 0); /* in order to avoid repetition of elements */ if (e0 > e1) diff --git a/phasta/phBubble.cc b/phasta/phBubble.cc index b155440c4..374662d26 100644 --- a/phasta/phBubble.cc +++ b/phasta/phBubble.cc @@ -1,4 +1,3 @@ -#include #include #include "phBubble.h" #include "phInput.h" @@ -17,14 +16,14 @@ struct Bubble { typedef std::vector Bubbles; -void readBubbles(Bubbles& bubbles, std::string bubbleFileName) +void readBubbles(Bubbles& bubbles, std::string bubbleFileName, pcu::PCU *PCUObj) { char bubblefname[1024]; FILE *filebubble; Bubble readbubble; snprintf(bubblefname, 1024, "%s", bubbleFileName.c_str()); - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) lion_oprint(1,"reading bubbles info from %s\n",bubblefname); filebubble = fopen(bubblefname, "r"); @@ -38,11 +37,11 @@ void readBubbles(Bubbles& bubbles, std::string bubbleFileName) { bubbles.push_back(readbubble); } - if(!feof(filebubble) && !PCU_Comm_Self()) // If while loop was exited for another reason then eof + if(!feof(filebubble) && !PCUObj->Self()) // If while loop was exited for another reason then eof lion_oprint(1,"WARNING: data in %s does not match expected format\n",bubblefname); fclose(filebubble); - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) lion_oprint(1,"%lu bubbles found in %s\n", bubbles.size(), bubblefname); } @@ -90,7 +89,7 @@ void setBubbleScalars(apf::Mesh* m, apf::MeshEntity* v, void initBubbles(apf::Mesh* m, Input& in) { Bubbles bubbles; - readBubbles(bubbles, in.bubbleFileName); + readBubbles(bubbles, in.bubbleFileName, m->getPCU()); PCU_ALWAYS_ASSERT(in.ensa_dof >= 7); apf::NewArray s(in.ensa_dof); apf::Field* f = m->findField("solution"); diff --git a/phasta/phCook.cc b/phasta/phCook.cc index 983570d43..6b168efca 100644 --- a/phasta/phCook.cc +++ b/phasta/phCook.cc @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -31,15 +30,15 @@ #include #include -static void print_stats(const char* name, double value) +static void print_stats(const char* name, double value, pcu::PCU *pcu_obj) { double min, max, avg; - min = PCU_Min_Double(value); - max = PCU_Max_Double(value); - avg = PCU_Add_Double(value); - avg /= PCU_Comm_Peers(); + min = pcu_obj->Min(value); + max = pcu_obj->Max(value); + avg = pcu_obj->Add(value); + avg /= pcu_obj->Peers(); double imb = max / avg; - if (!PCU_Comm_Self()) + if (!pcu_obj->Self()) printf("%s: min %f max %f avg %f imb %f\n", name, min, max, avg, imb); } @@ -47,47 +46,39 @@ static void print_stats(const char* name, double value) namespace { -void switchToMasters(int splitFactor) +pcu::PCU* createGroupComm(int splitFactor, pcu::PCU *PCUObj) { - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); int groupRank = self / splitFactor; int group = self % splitFactor; MPI_Comm groupComm; - MPI_Comm_split(PCU_Get_Comm(), group, groupRank, &groupComm); - PCU_Switch_Comm(groupComm); + MPI_Comm_split(PCUObj->GetMPIComm(), group, groupRank, &groupComm); + return new pcu::PCU(groupComm); } -void switchToAll(MPI_Comm orig) +void loadCommon(ph::Input& in, ph::BCs& bcs, gmi_model*& g, pcu::PCU *PCUObj) { - MPI_Comm prevComm = PCU_Get_Comm(); - PCU_Switch_Comm(orig); - MPI_Comm_free(&prevComm); - PCU_Barrier(); + ph::loadModelAndBCs(in, g, bcs, PCUObj); } -void loadCommon(ph::Input& in, ph::BCs& bcs, gmi_model*& g) -{ - ph::loadModelAndBCs(in, g, bcs); -} - -static apf::Mesh2* loadMesh(gmi_model*& g, ph::Input& in) { +static apf::Mesh2* loadMesh(gmi_model*& g, ph::Input& in, pcu::PCU *PCUObj) { const char* meshfile = in.meshFileName.c_str(); if (ph::mesh_has_ext(meshfile, "sms")) { if (in.simmetrixMesh == 0) { - if (PCU_Comm_Self()==0) + if (PCUObj->Self()==0) lion_eprint(1, "oops, turn on flag: simmetrixMesh\n"); in.simmetrixMesh = 1; in.filterMatches = 0; //not support } } - return ph::loadMesh(g, meshfile); + return ph::loadMesh(g, meshfile, PCUObj); } void originalMain(apf::Mesh2*& m, ph::Input& in, - gmi_model* g, apf::Migration*& plan) + gmi_model* g, apf::Migration*& plan, pcu::PCU *pcu_obj) { if(!m) - m = loadMesh(g, in); + m = loadMesh(g, in, pcu_obj); else apf::printStats(m); // Need to set a flag to enable avoiding this when short on time m->verify(); @@ -99,7 +90,7 @@ void originalMain(apf::Mesh2*& m, ph::Input& in, ph::attachZeroSolution(in, m); if (in.buildMapping) ph::buildMapping(m); - apf::setMigrationLimit(SIZET(in.elementsPerMigration)); + apf::setMigrationLimit(SIZET(in.elementsPerMigration), pcu_obj); if (in.adaptFlag) ph::adapt(in, m); if (in.tetrahedronize) @@ -111,30 +102,30 @@ void originalMain(apf::Mesh2*& m, ph::Input& in, }//end namespace namespace chef { - static FILE* openfile_read(ph::Input&, const char* path) { + static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU *PCUObj) { FILE* f = NULL; - PHASTAIO_OPENTIME(f = pcu_group_open(path, false);) + PHASTAIO_OPENTIME(f = pcu_group_open(PCUObj->GetCHandle(), path, false);) return f; } - static FILE* openfile_write(ph::Output&, const char* path) { + static FILE* openfile_write(ph::Output& out, const char* path) { FILE* f = NULL; - PHASTAIO_OPENTIME(f = pcu_group_open(path, true);) + PHASTAIO_OPENTIME(f = pcu_group_open(out.mesh->getPCU()->GetCHandle(), path, true);) return f; } static FILE* openstream_write(ph::Output& out, const char* path) { FILE* f = NULL; - PHASTAIO_OPENTIME(f = openGRStreamWrite(out.grs, path);) + PHASTAIO_OPENTIME(f = openGRStreamWrite(out.grs, path, out.mesh->getPCU());) return f; } - static FILE* openstream_read(ph::Input& in, const char* path) { + static FILE* openstream_read(ph::Input& in, const char* path, pcu::PCU *PCUObj) { std::string fname(path); std::string restartStr("restart"); FILE* f = NULL; if( fname.find(restartStr) != std::string::npos ) - PHASTAIO_OPENTIME(f = openRStreamRead(in.rs);) + PHASTAIO_OPENTIME(f = openRStreamRead(in.rs, PCUObj);) else { lion_eprint(1, "ERROR %s type of stream %s is unknown... exiting\n", @@ -149,13 +140,13 @@ namespace ph { void checkBalance(apf::Mesh2* m, ph::Input& in) { /* check if balancing was requested */ Parma_PrintPtnStats(m, "postSplit", false); - if (in.prePhastaBalanceMethod != "none" && PCU_Comm_Peers() > 1) + if (in.prePhastaBalanceMethod != "none" && m->getPCU()->Peers() > 1) ph::balance(in,m); } void checkReorder(apf::Mesh2* m, ph::Input& in, int numMasters) { /* check if the mesh changed at all */ - if ( (PCU_Comm_Peers()!=numMasters) || + if ( (m->getPCU()->Peers()!=numMasters) || in.splitFactor > 1 || in.adaptFlag || in.prePhastaBalanceMethod != "none" || @@ -164,16 +155,16 @@ namespace ph { { apf::MeshTag* order = NULL; - print_stats("malloc used before Bfs", PCU_GetMem()); + print_stats("malloc used before Bfs", pcu::GetMem(), m->getPCU()); - if (in.isReorder && PCU_Comm_Peers() > 1) + if (in.isReorder && m->getPCU()->Peers() > 1) order = Parma_BfsReorder(m); - print_stats("malloc used before reorder", PCU_GetMem()); + print_stats("malloc used before reorder", pcu::GetMem(), m->getPCU()); apf::reorderMdsMesh(m,order); - print_stats("malloc used after reorder", PCU_GetMem()); + print_stats("malloc used after reorder", pcu::GetMem(), m->getPCU()); } } @@ -184,22 +175,24 @@ namespace ph { } void preprocess(apf::Mesh2* m, Input& in, Output& out, BCs& bcs) { - phastaio_initStats(); - if(PCU_Comm_Peers() > 1) + PCU_t h; + h.ptr = static_cast(m->getPCU()); + phastaio_initStats(h); + if(m->getPCU()->Peers() > 1) ph::migrateInterfaceItr(m, bcs); if (in.simmetrixMesh == 0) - ph::checkReorder(m,in,PCU_Comm_Peers()); + ph::checkReorder(m,in,m->getPCU()->Peers()); if (in.adaptFlag) - ph::goToStepDir(in.timeStepNumber,in.ramdisk); - std::string path = ph::setupOutputDir(in.ramdisk); + ph::goToStepDir(in.timeStepNumber, m->getPCU(), in.ramdisk); + std::string path = ph::setupOutputDir(m->getPCU(), in.ramdisk); std::string subDirPath = path; - ph::setupOutputSubdir(subDirPath,in.ramdisk); + ph::setupOutputSubdir(subDirPath, m->getPCU(), in.ramdisk); ph::enterFilteredMatching(m, in, bcs); ph::generateOutput(in, bcs, m, out); ph::exitFilteredMatching(m); // a path is not needed for inmem if ( in.writeRestartFiles ) { - if(!PCU_Comm_Self()) lion_oprint(1,"write file-based restart file\n"); + if(!m->getPCU()->Self()) lion_oprint(1,"write file-based restart file\n"); // store the value of the function pointer FILE* (*fn)(Output& out, const char* path) = out.openfile_write; // set function pointer for file writing @@ -214,7 +207,7 @@ namespace ph { if ( ! in.outMeshFileName.empty() ) m->writeNative(in.outMeshFileName.c_str()); if ( in.writeGeomBCFiles ) { - if(!PCU_Comm_Self()) lion_oprint(1,"write additional geomBC file for visualization\n"); + if(!m->getPCU()->Self()) lion_oprint(1,"write additional geomBC file for visualization\n"); // store the value of the function pointer FILE* (*fn)(Output& out, const char* path) = out.openfile_write; // set function pointer for file writing @@ -224,8 +217,8 @@ namespace ph { out.openfile_write = fn; } ph::writeGeomBC(out, subDirPath); //write geombc - if(!PCU_Comm_Self()) - ph::writeAuxiliaryFiles(path, in.timeStepNumber); + if(!m->getPCU()->Self()) + ph::writeAuxiliaryFiles(path, in.timeStepNumber, m->getPCU()); m->verify(); #ifdef HAVE_SIMMETRIX gmi_model* g = m->getModel(); @@ -233,7 +226,7 @@ namespace ph { #endif if (in.adaptFlag) ph::goToParentDir(); - if(in.printIOtime) phastaio_printStats(); + if(in.printIOtime) phastaio_printStats(h); } void preprocess(apf::Mesh2* m, Input& in, Output& out) { gmi_model* g = m->getModel(); @@ -266,27 +259,25 @@ namespace { namespace chef { void bake(gmi_model*& g, apf::Mesh2*& m, - ph::Input& in, ph::Output& out) { + ph::Input& in, ph::Output& out, pcu::PCU *expandedPCUObj) { int shrinkFactor=0; if(in.splitFactor < 0) { shrinkFactor=-1*in.splitFactor; in.splitFactor=1; // this is used in to set readers so if shrinking need to read all } - PCU_ALWAYS_ASSERT(PCU_Comm_Peers() % in.splitFactor == 0); + PCU_ALWAYS_ASSERT(expandedPCUObj->Peers() % in.splitFactor == 0); apf::Migration* plan = 0; ph::BCs bcs; - loadCommon(in, bcs, g); - const int worldRank = PCU_Comm_Self(); - MPI_Comm comm = PCU_Get_Comm(); - switchToMasters(in.splitFactor); + loadCommon(in, bcs, g, expandedPCUObj); + const int worldRank = expandedPCUObj->Self(); + pcu::PCU *groupPCUObj = createGroupComm(in.splitFactor, expandedPCUObj); if ((worldRank % in.splitFactor) == 0) - originalMain(m, in, g, plan); - switchToAll(comm); + originalMain(m, in, g, plan, groupPCUObj); if (in.simmetrixMesh == 0) - m = repeatMdsMesh(m, g, plan, in.splitFactor); + m = repeatMdsMesh(m, g, plan, in.splitFactor, expandedPCUObj); if (in.simmetrixMesh == 0 && shrinkFactor > 1){ GroupCode code; - apf::Unmodulo outMap(PCU_Comm_Self(), PCU_Comm_Peers()); + apf::Unmodulo outMap(expandedPCUObj->Self(), expandedPCUObj->Peers()); code.mesh=m; code.input=∈ code.boundary=&bcs; @@ -296,48 +287,48 @@ namespace chef { ph::preprocess(m,in,out,bcs); } } - void cook(gmi_model*& g, apf::Mesh2*& m) { + void cook(gmi_model*& g, apf::Mesh2*& m, pcu::PCU *pcu_obj) { ph::Input in; - in.load("adapt.inp"); + in.load("adapt.inp", pcu_obj); in.openfile_read = openfile_read; ph::Output out; out.openfile_write = openfile_write; - bake(g,m,in,out); + bake(g,m,in,out,pcu_obj); if ((in.writeVTK) == 1) apf::writeVtkFiles("rendered",m); } void cook(gmi_model*& g, apf::Mesh2*& m, - ph::Input& ctrl) { + ph::Input& ctrl, pcu::PCU *pcu_obj) { ctrl.openfile_read = openfile_read; ph::Output out; out.openfile_write = openfile_write; - bake(g,m,ctrl,out); + bake(g,m,ctrl,out,pcu_obj); } void cook(gmi_model*& g, apf::Mesh2*& m, - ph::Input& ctrl, GRStream* grs) { + ph::Input& ctrl, GRStream* grs, pcu::PCU *pcu_obj) { ctrl.openfile_read = openfile_read; ph::Output out; out.openfile_write = openstream_write; out.grs = grs; - bake(g,m,ctrl,out); + bake(g,m,ctrl,out,pcu_obj); } void cook(gmi_model*& g, apf::Mesh2*& m, - ph::Input& ctrl, RStream* rs) { + ph::Input& ctrl, RStream* rs, pcu::PCU *pcu_obj) { ctrl.openfile_read = openstream_read; ctrl.rs = rs; ph::Output out; out.openfile_write = openfile_write; - bake(g,m,ctrl,out); + bake(g,m,ctrl,out,pcu_obj); return; } void cook(gmi_model*& g, apf::Mesh2*& m, - ph::Input& ctrl, RStream* rs, GRStream* grs) { + ph::Input& ctrl, RStream* rs, GRStream* grs, pcu::PCU *pcu_obj) { ctrl.openfile_read = openstream_read; ctrl.rs = rs; ph::Output out; out.openfile_write = openstream_write; out.grs = grs; - bake(g,m,ctrl,out); + bake(g,m,ctrl,out,pcu_obj); return; } @@ -365,7 +356,7 @@ namespace chef { } void balanceAndReorder(ph::Input& ctrl, apf::Mesh2* m) { - ph::balanceAndReorder(m,ctrl,PCU_Comm_Peers()); + ph::balanceAndReorder(m,ctrl,m->getPCU()->Peers()); } void balance(ph::Input& ctrl, apf::Mesh2* m) { @@ -384,5 +375,4 @@ namespace chef { out.grs = grs; ph::preprocess(m,in,out); } -} - +} \ No newline at end of file diff --git a/phasta/phFilterMatching.cc b/phasta/phFilterMatching.cc index 0aa5f4170..a81bb7093 100644 --- a/phasta/phFilterMatching.cc +++ b/phasta/phFilterMatching.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -225,7 +224,7 @@ void filterMatching(apf::Mesh2* m, ModelMatching& mm, int dim) { gmi_model* gm; gm = m->getModel(); - PCU_Comm_Begin(); + m->getPCU()->Begin(); apf::MeshIterator* it = m->begin(dim); apf::MeshEntity* e; int gd, gt; @@ -238,28 +237,28 @@ void filterMatching(apf::Mesh2* m, ModelMatching& mm, int dim) gd = gmi_dim(gm, ge); gt = gmi_tag(gm, ge); APF_ITERATE(apf::Matches, matches, mit) { - PCU_COMM_PACK(mit->peer, mit->entity); - PCU_COMM_PACK(mit->peer, e); - PCU_COMM_PACK(mit->peer, gd); - PCU_COMM_PACK(mit->peer, gt); + m->getPCU()->Pack(mit->peer, mit->entity); + m->getPCU()->Pack(mit->peer, e); + m->getPCU()->Pack(mit->peer, gd); + m->getPCU()->Pack(mit->peer, gt); } m->clearMatches(e); } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { - PCU_COMM_UNPACK(e); + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { + m->getPCU()->Unpack(e); apf::MeshEntity* oe; - PCU_COMM_UNPACK(oe); - PCU_COMM_UNPACK(gd); - PCU_COMM_UNPACK(gt); + m->getPCU()->Unpack(oe); + m->getPCU()->Unpack(gd); + m->getPCU()->Unpack(gt); gmi_ent* ge = (gmi_ent*) m->toModel(e); if (!mm.count(ge)) continue; ModelSet& ms = mm[ge]; gmi_ent* oge = gmi_find(gm, gd, gt); if (oge == ge || ms.count(oge)) - m->addMatch(e, PCU_Comm_Sender(), oe); + m->addMatch(e, m->getPCU()->Sender(), oe); } checkFilteredMatching(m, mm, dim); } diff --git a/phasta/phGeomBC.cc b/phasta/phGeomBC.cc index c61b4064c..8b5aaa342 100644 --- a/phasta/phGeomBC.cc +++ b/phasta/phGeomBC.cc @@ -1,4 +1,3 @@ -#include #include "phOutput.h" #include "phIO.h" #include "phiotimer.h" @@ -9,10 +8,10 @@ namespace ph { -static std::string buildGeomBCFileName(std::string timestep_or_dat) +static std::string buildGeomBCFileName(std::string timestep_or_dat, pcu::PCU *pcu_obj) { std::stringstream ss; - int rank = PCU_Comm_Self() + 1; + int rank = pcu_obj->Self() + 1; ss << "geombc." << timestep_or_dat << "." << rank; return ss.str(); } @@ -294,7 +293,7 @@ static void writeSpanwiseAvgArrays(Output& o, FILE* f) void writeGeomBC(Output& o, std::string path, int timestep) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); apf::Mesh* m = o.mesh; std::stringstream tss; std::string timestep_or_dat; @@ -304,7 +303,7 @@ void writeGeomBC(Output& o, std::string path, int timestep) tss << timestep; timestep_or_dat = tss.str(); } - path += buildGeomBCFileName(timestep_or_dat); + path += buildGeomBCFileName(timestep_or_dat, m->getPCU()); phastaio_setfile(GEOMBC_WRITE); FILE* f = o.openfile_write(o, path.c_str()); if (!f) { @@ -354,7 +353,7 @@ void writeGeomBC(Output& o, std::string path, int timestep) params[1] = 3; ph_write_doubles(f, "co-ordinates", o.arrays.coordinates, params[0] * params[1], 2, params); - writeInt(f, "number of processors", PCU_Comm_Peers()); + writeInt(f, "number of processors", m->getPCU()->Peers()); writeInt(f, "size of ilwork array", o.nlwork); if (o.nlwork) writeInts(f, "ilwork ", o.arrays.ilwork, o.nlwork); @@ -373,8 +372,8 @@ void writeGeomBC(Output& o, std::string path, int timestep) writeGrowthCurves(o, f); writeSpanwiseAvgArrays(o, f); PHASTAIO_CLOSETIME(fclose(f);) - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) lion_oprint(1,"geombc file written in %f seconds\n", t1 - t0); } diff --git a/phasta/phGrowthCurves.cc b/phasta/phGrowthCurves.cc index d8bef4e65..b61d74cd3 100644 --- a/phasta/phGrowthCurves.cc +++ b/phasta/phGrowthCurves.cc @@ -1,4 +1,3 @@ -#include #include #include #include "phOutput.h" @@ -236,12 +235,12 @@ void getGrowthCurves(Output& o) o.arrays.igclv[i] = me; } - lion_oprint(1,"%s: rank %d, ngc, nv: %d, %d\n", __func__, PCU_Comm_Self(), ngc, nv); + lion_oprint(1,"%s: rank %d, ngc, nv: %d, %d\n", __func__, o.mesh->getPCU()->Self(), ngc, nv); - PCU_Add_Ints(&ngc,sizeof(ngc)); - PCU_Add_Ints(&nv,sizeof(nv)); + o.mesh->getPCU()->Add(&ngc,sizeof(ngc)); + o.mesh->getPCU()->Add(&nv,sizeof(nv)); - if(PCU_Comm_Self() == 0) + if(o.mesh->getPCU()->Self() == 0) lion_oprint(1,"%s: total ngc, nv: %d, %d\n", __func__, ngc, nv); PList_delete(gEdges); @@ -261,7 +260,7 @@ void getGrowthCurves(Output& o) Sim_logOff(); } else { - if(PCU_Comm_Self() == 0) + if(o.mesh->getPCU()->Self() == 0) lion_oprint(1,"%s: warning! not implemented for MDS mesh\n",__func__); } return; diff --git a/phasta/phGrowthCurves_empty.cc b/phasta/phGrowthCurves_empty.cc index c37eb6874..297bd5649 100644 --- a/phasta/phGrowthCurves_empty.cc +++ b/phasta/phGrowthCurves_empty.cc @@ -1,4 +1,3 @@ -#include #include #include "phGrowthCurves.h" #include "phOutput.h" @@ -6,7 +5,7 @@ namespace ph { void getGrowthCurves(Output& o) { o.nGrowthCurves = 0; o.nLayeredMeshVertices = 0; - if(PCU_Comm_Self() == 0) + if(o.mesh->getPCU()->Self() == 0) lion_oprint(1,"warning! \'%s\' requires the Simmetrix SimAdvMeshing library\n",__func__); return; } diff --git a/phasta/phIO.c b/phasta/phIO.c index 2806dd783..caf3c590a 100644 --- a/phasta/phIO.c +++ b/phasta/phIO.c @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -70,7 +70,7 @@ static void parse_header(char* header, char** name, long* bytes, } } -static int find_header(FILE* f, const char* name, char* found, char header[PH_LINE]) +static int find_header(PCU_t h, FILE* f, const char* name, char* found, char header[PH_LINE]) { char* hname; long bytes; @@ -88,7 +88,7 @@ static int find_header(FILE* f, const char* name, char* found, char header[PH_LI } fseek(f, bytes, SEEK_CUR); } - if (!PCU_Comm_Self() && strlen(name) > 0) + if (!PCU_Comm_Self(h) && strlen(name) > 0) lion_eprint(1,"warning: phIO could not find \"%s\"\n",name); return 0; } @@ -102,11 +102,11 @@ static void write_magic_number(FILE* f) fprintf(f,"\n"); } -static int seek_after_header(FILE* f, const char* name) +static int seek_after_header(PCU_t h, FILE* f, const char* name) { char dummy[PH_LINE]; char found[PH_LINE]; - return find_header(f, name, found, dummy); + return find_header(h, f, name, found, dummy); } static void my_fread(void* p, size_t size, size_t nmemb, FILE* f) @@ -115,11 +115,11 @@ static void my_fread(void* p, size_t size, size_t nmemb, FILE* f) PCU_ALWAYS_ASSERT(r == nmemb); } -static int read_magic_number(FILE* f) +static int read_magic_number(PCU_t h, FILE* f) { int magic; - if (!seek_after_header(f, magic_name)) { - if (!PCU_Comm_Self()) + if (!seek_after_header(h, f, magic_name)) { + if (!PCU_Comm_Self(h)) lion_eprint(1,"warning: not swapping bytes\n"); rewind(f); return 0; @@ -162,17 +162,17 @@ static void parse_params(char* header, long* bytes, *step = params[STEP_PARAM]; } -int ph_should_swap(FILE* f) { - return read_magic_number(f); +int ph_should_swap(FILE* f, PCU_t h) { + return read_magic_number(h, f); } -int ph_read_field(FILE* f, const char* field, int swap, +int ph_read_field(FILE* f, PCU_t h, const char* field, int swap, double** data, int* nodes, int* vars, int* step, char* hname) { long bytes, n; char header[PH_LINE]; int ok; - ok = find_header(f, field, hname, header); + ok = find_header(h, f, field, hname, header); if(!ok) /* not found */ return 0; parse_params(header, &bytes, nodes, vars, step); diff --git a/phasta/phIO.h b/phasta/phIO.h index 4c9bbebbe..4a53ead2f 100644 --- a/phasta/phIO.h +++ b/phasta/phIO.h @@ -18,14 +18,14 @@ void ph_write_ints(FILE* f, const char* name, int* data, * swapped to account for endianness * @return 1 if swapping is required, 0 otherwise */ -int ph_should_swap(FILE* f); +int ph_should_swap(FILE* f, PCU_t h); /** * @brief read a field * @return 1 if no data block was read, 2 if data block read, 0 otherwise */ -int ph_read_field(FILE* f, const char* field, int swap, +int ph_read_field(FILE* f, PCU_t h, const char* field, int swap, double** data, int* nodes, int* vars, int* step, char* hname); void ph_write_field(FILE* f, const char* field, double* data, int nodes, int vars, int step); diff --git a/phasta/phInput.cc b/phasta/phInput.cc index ffc9989c2..f34f0d9af 100644 --- a/phasta/phInput.cc +++ b/phasta/phInput.cc @@ -1,4 +1,3 @@ -#include #include "phInput.h" #include #include @@ -210,10 +209,10 @@ static void makeDeprecated(stringset& old) old.insert("writePhastaFiles"); } -static bool deprecated(stringset& old, std::string const& name) +static bool deprecated(stringset& old, std::string const& name, pcu::PCU *PCUObj) { if( old.count(name) ) { - if( !PCU_Comm_Self() ) + if( !PCUObj->Self() ) lion_eprint(1, "WARNING deprecated input \"%s\" ... " "carefully check stderr and stdout for unexpected behavior\n", name.c_str()); @@ -227,7 +226,8 @@ static void readInputFile( const char* filename, StringMap& stringMap, IntMap& intMap, - DblMap& dblMap) + DblMap& dblMap, + pcu::PCU *PCUObj) { stringset old; makeDeprecated(old); @@ -236,7 +236,7 @@ static void readInputFile( fail("could not open \"%s\"", filename); std::string name; while (f >> name) { - if (name[0] == '#' || deprecated(old,name)) { + if (name[0] == '#' || deprecated(old,name,PCUObj)) { std::getline(f, name, '\n'); continue; } @@ -257,14 +257,14 @@ static void validate(Input& in) PCU_ALWAYS_ASSERT( ! (in.buildMapping && in.adaptFlag)); } -void Input::load(const char* filename) +void Input::load(const char* filename, pcu::PCU *PCUObj) { setDefaults(*this); StringMap stringMap; IntMap intMap; DblMap dblMap; formMaps(*this, stringMap, intMap, dblMap); - readInputFile(filename, stringMap, intMap, dblMap); + readInputFile(filename, stringMap, intMap, dblMap, PCUObj); validate(*this); } diff --git a/phasta/phInput.h b/phasta/phInput.h index a6bf88c90..9b823835a 100644 --- a/phasta/phInput.h +++ b/phasta/phInput.h @@ -13,6 +13,10 @@ #include #include +namespace pcu{ + class PCU; +} + struct RStream; namespace ph { @@ -22,7 +26,7 @@ class Input { public: Input(); - void load(const char* filename); + void load(const char* filename, pcu::PCU *PCUObj); int txtCoord; //HACK added to get through compile int timeStepNumber; /** \brief this corresponds to the number of degrees of @@ -158,7 +162,7 @@ class Input double meshqCrtn; double elementImbalance; double vertexImbalance; - FILE* (*openfile_read)(Input& in, const char* path); + FILE* (*openfile_read)(Input& in, const char* path, pcu::PCU *PCUObj); RStream* rs; /** \brief the flag for switch between simmetrix mesh and pumi-based mesh. avoid run incompatible APIs with simmetrix mesh */ @@ -216,5 +220,3 @@ int countScalarBCs(Input& in); } #endif - - diff --git a/phasta/phInterfaceCutter.cc b/phasta/phInterfaceCutter.cc index 90fe0d2dc..76d5513ea 100644 --- a/phasta/phInterfaceCutter.cc +++ b/phasta/phInterfaceCutter.cc @@ -1,6 +1,5 @@ #include "phInterfaceCutter.h" #include -#include #include #include #include @@ -262,18 +261,18 @@ int migrateInterface(apf::Mesh2*& m, ph::BCs& bcs) { int remoteResidence = -1; for (size_t j = 0; j != dgCopies.getSize(); ++j) { - if (dgCopies[j].peer != PCU_Comm_Self()) + if (dgCopies[j].peer != m->getPCU()->Self()) remoteResidence = dgCopies[j].peer; } - if (remoteResidence > PCU_Comm_Self()) + if (remoteResidence > m->getPCU()->Self()) plan->send(e,remoteResidence); } m->end(it); - lion_oprint(1,"proc-%d: number of migrating elements: %d\n",PCU_Comm_Self(),plan->count()); + lion_oprint(1,"proc-%d: number of migrating elements: %d\n",m->getPCU()->Self(),plan->count()); int totalPlan = plan->count(); - totalPlan = PCU_Add_Int(totalPlan); + totalPlan = m->getPCU()->Add(totalPlan); m->migrate(plan); return totalPlan; diff --git a/phasta/phLinks.cc b/phasta/phLinks.cc index 683bf2c1e..62b72f7d1 100644 --- a/phasta/phLinks.cc +++ b/phasta/phLinks.cc @@ -1,9 +1,9 @@ -#include #include "phLinks.h" #include "phAdjacent.h" #include #include #include +#include namespace ph { @@ -24,7 +24,7 @@ struct PhastaSharing : public apf::Sharing { helperN = new apf::NormalSharing(m); helperM = new apf::MatchedSharing(m); } - ~PhastaSharing() + virtual ~PhastaSharing() { delete helperN; delete helperM; @@ -54,7 +54,7 @@ struct PhastaSharing : public apf::Sharing { if ( ! mesh->hasMatching()) return; /* filter out matches which are on the same part as the global master */ - int self = PCU_Comm_Self(); + int self = mesh->getPCU()->Self(); size_t i = 0; for (size_t j = 0; j < copies.getSize(); ++j) if (copies[j].peer != self) @@ -98,8 +98,9 @@ struct PhastaSharing : public apf::Sharing { void getLinks(apf::Mesh* m, int dim, Links& links, BCs& bcs) { - PhastaSharing* shr = new PhastaSharing(m); - PCU_Comm_Begin(); + //PhastaSharing* shr = new PhastaSharing(m); + auto shr = std::unique_ptr(new PhastaSharing(m)); + m->getPCU()->Begin(); apf::MeshIterator* it = m->begin(dim); apf::MeshEntity* v; while ((v = m->iterate(it))) { @@ -118,20 +119,19 @@ void getLinks(apf::Mesh* m, int dim, Links& links, BCs& bcs) of the same master in the outgoing links array to a part that contains multiple copies of it. */ links[LinkKey(1, remotes[i].peer)].push_back(v); - PCU_COMM_PACK(remotes[i].peer, remotes[i].entity); + m->getPCU()->Pack(remotes[i].peer, remotes[i].entity); } } m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Listen()) { - int peer = PCU_Comm_Sender(); - while (!PCU_Comm_Unpacked()) { + m->getPCU()->Send(); + while (m->getPCU()->Listen()) { + int peer = m->getPCU()->Sender(); + while (!m->getPCU()->Unpacked()) { apf::MeshEntity* v; - PCU_COMM_UNPACK(v); + m->getPCU()->Unpack(v); links[LinkKey(0, peer)].push_back(v); } } - delete shr; } /* encode the local links into a big array of integers @@ -218,7 +218,7 @@ static apf::MeshEntity* getOtherElem(apf::Mesh* m, apf::MeshEntity* elem, return 0; apf::Matches matches; m->getMatches(face, matches); - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); for (size_t i = 0; i < matches.getSize(); ++i) if (matches[i].peer == self) return m->getUpward(matches[i].entity, 0); diff --git a/phasta/phMeshQuality.cc b/phasta/phMeshQuality.cc index 3a21e315c..c96ac84ff 100644 --- a/phasta/phMeshQuality.cc +++ b/phasta/phMeshQuality.cc @@ -19,7 +19,6 @@ #endif #endif -#include #include #include #include diff --git a/phasta/phOutput.cc b/phasta/phOutput.cc index d4b71028b..65bd3905f 100644 --- a/phasta/phOutput.cc +++ b/phasta/phOutput.cc @@ -1,4 +1,3 @@ -#include #include #include "phOutput.h" #include "phGrowthCurves.h" @@ -32,23 +31,23 @@ static void getCounts(Output& o) static void checkLoadBalance(Output& o) { - long sumOwnedNodes = PCU_Add_Long(o.nOwnedNodes); - long sumAllNodes = PCU_Add_Long(o.nOverlapNodes); - double avgNodes = static_cast(sumAllNodes) / PCU_Comm_Peers(); + long sumOwnedNodes = o.mesh->getPCU()->Add(o.nOwnedNodes); + long sumAllNodes = o.mesh->getPCU()->Add(o.nOverlapNodes); + double avgNodes = static_cast(sumAllNodes) / o.mesh->getPCU()->Peers(); double vlbratio = o.nOverlapNodes / avgNodes; - double vlbratio_max = PCU_Max_Double(vlbratio); - if (!PCU_Comm_Self()) + double vlbratio_max = o.mesh->getPCU()->Max(vlbratio); + if (!o.mesh->getPCU()->Self()) lion_oprint(1,"max vertex load imbalance of partitioned mesh = %f\n", vlbratio_max); - if (!PCU_Comm_Self()) + if (!o.mesh->getPCU()->Self()) lion_oprint(1,"ratio of sum of all vertices to sum of owned vertices = %f\n", sumAllNodes / (double) sumOwnedNodes); int dim = o.mesh->getDimension(); int numElms = o.mesh->count(dim); - long sumElms = PCU_Add_Long(numElms); - double avgElms = static_cast(sumElms) / PCU_Comm_Peers(); + long sumElms = o.mesh->getPCU()->Add(numElms); + double avgElms = static_cast(sumElms) / o.mesh->getPCU()->Peers(); double elbratio = numElms / avgElms; - double elbratio_max = PCU_Max_Double(elbratio); - if (!PCU_Comm_Self()) + double elbratio_max = o.mesh->getPCU()->Max(elbratio); + if (!o.mesh->getPCU()->Self()) lion_oprint(1,"max region (3D) or face (2D) load imbalance of partitioned mesh = %f\n", elbratio_max); } @@ -115,8 +114,8 @@ static void getGlobal(Output& o) { apf::Mesh* m = o.mesh; int n = m->count(0); - int self = PCU_Comm_Self(); - int peers = PCU_Comm_Peers(); + int self = m->getPCU()->Self(); + int peers = m->getPCU()->Peers(); int id = self + 1; o.arrays.globalNodeNumbers = new int[n]; for (int i = 0; i < n; ++i) { @@ -299,7 +298,7 @@ static void getBoundary(Output& o, BCs& bcs, apf::Numbering* n) } static void getRigidBody(Output& o, BCs& bcs, apf::Numbering* n) { - PCU_Comm_Begin(); + o.mesh->getPCU()->Begin(); apf::Mesh* m = o.mesh; gmi_model* gm = m->getModel(); int rbID = 0; // id - set by user @@ -343,8 +342,8 @@ static void getRigidBody(Output& o, BCs& bcs, apf::Numbering* n) { rit = rbIDmap.find(rbID); if(rit == rbIDmap.end()) { rbIDmap[rbID] = rbMT; - PCU_Comm_Pack(0, &rbID, sizeof(int)); - PCU_Comm_Pack(0, &rbMT, sizeof(int)); + m->getPCU()->Pack(0, &rbID, sizeof(int)); + m->getPCU()->Pack(0, &rbMT, sizeof(int)); } int vID = apf::getNumber(n, e, 0, 0); if(f[vID] > -1 && f[vID] != rbID) { @@ -359,22 +358,22 @@ static void getRigidBody(Output& o, BCs& bcs, apf::Numbering* n) { } // end if rigid body attribute exists // master receives and form the complete set - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { int rrbID = 0; int rrbMT = 0; - PCU_Comm_Unpack(&rrbID, sizeof(int)); - PCU_Comm_Unpack(&rrbMT, sizeof(int)); + m->getPCU()->Unpack(&rrbID, sizeof(int)); + m->getPCU()->Unpack(&rrbMT, sizeof(int)); rit = rbIDmap.find(rrbID); if(rit == rbIDmap.end()) rbIDmap[rrbID] = rrbMT; } - int rbIDs_size = PCU_Max_Int(rbIDmap.size()); + int rbIDs_size = m->getPCU()->Max(rbIDmap.size()); int* rbIDs = new int[rbIDs_size](); int* rbMTs = new int[rbIDs_size](); - if (!PCU_Comm_Self()) { + if (!m->getPCU()->Self()) { int count = 0; for (rit=rbIDmap.begin(); rit!=rbIDmap.end(); rit++) { rbIDs[count] = rit->first; @@ -385,8 +384,8 @@ static void getRigidBody(Output& o, BCs& bcs, apf::Numbering* n) { } // allreduce the set - PCU_Max_Ints(rbIDs, rbIDs_size); - PCU_Max_Ints(rbMTs, rbIDs_size); + m->getPCU()->Max(rbIDs, rbIDs_size); + m->getPCU()->Max(rbMTs, rbIDs_size); // attach data o.numRigidBody = rbIDs_size; @@ -434,7 +433,7 @@ bool checkInterface(Output& o, BCs& bcs) { m->end(it); PCU_ALWAYS_ASSERT(aID!=bID); //assert different material ID on two sides PCU_ALWAYS_ASSERT(a==b); //assert same number of faces on each side - if (PCU_Comm_Self() == 0) + if (m->getPCU()->Self() == 0) lion_oprint(1,"Checked! Same number of faces on each side of interface.\n"); return true; } @@ -627,12 +626,12 @@ static void getMaxElementNodes(Output& o) /* returns the global periodic master iff it is on this part, otherwise returns e */ static apf::MeshEntity* getLocalPeriodicMaster(apf::MatchedSharing* sh, - apf::MeshEntity* e) + apf::MeshEntity* e, pcu::PCU *PCUObj) { if ( ! sh) return e; apf::Copy globalMaster = sh->getOwnerCopy(e); - if (globalMaster.peer == PCU_Comm_Self()) + if (globalMaster.peer == PCUObj->Self()) return globalMaster.entity; else return e; @@ -649,7 +648,7 @@ static void getLocalPeriodicMasters(Output& o, apf::Numbering* n, BCs& bcs) while ((e = m->iterate(it))) { apf::ModelEntity* me = m->toModel(e); bool isDG = ph::isInterface(m->getModel(),(gmi_ent*) me,bcs.fields["DG interface"]); - apf::MeshEntity* master = getLocalPeriodicMaster(sh, e); + apf::MeshEntity* master = getLocalPeriodicMaster(sh, e, m->getPCU()); if (master == e || isDG) iper[i] = 0; else @@ -739,7 +738,7 @@ static void getGCEssentialBCs(Output& o, apf::Numbering* n) apf::Mesh* m = o.mesh; if(!in.ensa_melas_dof) return; - PCU_Comm_Begin(); + m->getPCU()->Begin(); int nec = countEssentialBCs(in); int& ei = o.nEssentialBCNodes; @@ -793,9 +792,9 @@ static void getGCEssentialBCs(Output& o, apf::Numbering* n) if(j == igcnv - 1 && m->isShared(vent)){ m->getRemotes(vent, remotes); APF_ITERATE(apf::Copies, remotes, rit) { - PCU_COMM_PACK(rit->first, rit->second); - PCU_Comm_Pack(rit->first, &ibc, sizeof(int)); - PCU_Comm_Pack(rit->first, &(bbc[0]), nec*sizeof(double)); + m->getPCU()->Pack(rit->first, rit->second); + m->getPCU()->Pack(rit->first, &ibc, sizeof(int)); + m->getPCU()->Pack(rit->first, &(bbc[0]), nec*sizeof(double)); } } } // end loop over nodes on a growth curve @@ -803,14 +802,14 @@ static void getGCEssentialBCs(Output& o, apf::Numbering* n) } // receive top most node - PCU_Comm_Send(); - while (PCU_Comm_Receive()) { + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { apf::MeshEntity* rvent; - PCU_COMM_UNPACK(rvent); + m->getPCU()->Unpack(rvent); int ribc = 0; - PCU_Comm_Unpack(&ribc, sizeof(int)); + m->getPCU()->Unpack(&ribc, sizeof(int)); double* rbc = new double[nec]; - PCU_Comm_Unpack(&(rbc[0]), nec*sizeof(double)); + m->getPCU()->Unpack(&(rbc[0]), nec*sizeof(double)); vID = apf::getNumber(n, rvent, 0, 0); if(o.arrays.nbc[vID] <= 0){ o.arrays.nbc[vID] = ei + 1; @@ -841,7 +840,7 @@ static void getInitialConditions(BCs& bcs, Output& o) { Input& in = *o.in; if (in.solutionMigration) { - if (!PCU_Comm_Self()) + if (!o.mesh->getPCU()->Self()) lion_oprint(1,"All attribute-based initial conditions, " "if any, " "are ignored due to request for SolutionMigration\n"); @@ -967,10 +966,10 @@ static void getSpanwiseAverageArrays(Input& in, Output& o) { o.arrays.ifather = new int[nnodes]; //initialize ifath apf::MeshTag* t = m->findTag("fathers2D"); if (t==NULL) { - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_oprint(1,"Did not find tag fathers2D\n"); } else if (t != NULL) { - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_oprint(1,"Found tag fathers2D\n"); } int tagNum; @@ -1073,7 +1072,7 @@ Output::~Output() void generateOutput(Input& in, BCs& bcs, apf::Mesh* mesh, Output& o) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); o.in = ∈ o.mesh = mesh; getCounts(o); @@ -1107,8 +1106,8 @@ void generateOutput(Input& in, BCs& bcs, apf::Mesh* mesh, Output& o) apf::destroyNumbering(rn); if (in.initBubbles) initBubbles(o.mesh, in); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!o.mesh->getPCU()->Self()) lion_oprint(1,"generated output structs in %f seconds\n",t1 - t0); } diff --git a/phasta/phPartition.cc b/phasta/phPartition.cc index 8546c4188..0d310e8ca 100644 --- a/phasta/phPartition.cc +++ b/phasta/phPartition.cc @@ -1,4 +1,3 @@ -#include #include "phPartition.h" #include "phInput.h" #include "ph.h" @@ -94,7 +93,7 @@ bool isMixed(apf::Mesh2* m) { break; } m->end(it); - return PCU_Max_Int(mixed); + return m->getPCU()->Max(mixed); } void clearTags(apf::Mesh* m, apf::MeshTag* t) { @@ -144,7 +143,7 @@ void parmaTet(Input& in, apf::Mesh2* m, bool runGap) { neighborReduction(m,weights,verbose,fineStats); double vtxImb = Parma_GetWeightedEntImbalance(m, weights, 0); if( vtxImb <= in.vertexImbalance ) { - if( !PCU_Comm_Self() ) + if( !m->getPCU()->Self() ) lion_oprint(1, "STATUS vtx imbalance target %.3f reached\n", in.vertexImbalance); break; @@ -206,7 +205,7 @@ void simmetrixBalance(apf::Mesh2* m) // current total num parts in pmesh cannot be more than requested int currentTotalNumParts = PM_totalNumParts(pmesh); if (currentTotalNumParts > totalNumParts) { - if( !PCU_Comm_Self() ) + if( !m->getPCU()->Self() ) lion_eprint(1, "Error: cannot reduce number of partitions %d->%d\n", currentTotalNumParts, totalNumParts); totalNumParts = currentTotalNumParts; diff --git a/phasta/phRestart.cc b/phasta/phRestart.cc index 2135f2616..c5fec0650 100644 --- a/phasta/phRestart.cc +++ b/phasta/phRestart.cc @@ -1,4 +1,3 @@ -#include #include #include "phRestart.h" #include @@ -29,7 +28,7 @@ apf::Field* extractField(apf::Mesh* m, bool simField) { apf::Field* f = m->findField(packedFieldname); - if(!f && PCU_Comm_Self() == 0) + if(!f && m->getPCU()->Self() == 0) lion_eprint(1, "No packed field \"%s\"", packedFieldname); PCU_ALWAYS_ASSERT(f); apf::Field* rf = m->findField(requestFieldname); @@ -321,7 +320,7 @@ static bool isNodalField(const char* fieldname, int nnodes, apf::Mesh* m) for (int i = 0; i < known_rand_field_count; ++i) if (!strcmp(fieldname, known_rand_fields[i])) return false; - if( !PCU_Comm_Self() ) { + if( !m->getPCU()->Self() ) { lion_eprint(1, "unknown restart field name \"%s\"\n", fieldname); lion_eprint(1, "please add \"%s\" to isNodalField above line %d of %s\n", fieldname, __LINE__, __FILE__); @@ -344,8 +343,8 @@ int readAndAttachField( int nodes, vars, step; char hname[1024]; const char* anyfield = ""; - int ret = ph_read_field(f, anyfield, swap, - &data, &nodes, &vars, &step, hname); + int ret = ph_read_field(f, m->getPCU()->GetCHandle(), + anyfield, swap, &data, &nodes, &vars, &step, hname); /* no field was found or the field has an empty data block */ if(ret==0 || ret==1) return ret; @@ -363,7 +362,7 @@ int readAndAttachField( if ( std::string(hname) == std::string("solution") ) out_size = in.ensa_dof; if (m->findField(hname)) { - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_eprint(1, "field \"%s\" already attached to the mesh, " "ignoring request to re-attach...\n", hname); } else { @@ -443,7 +442,7 @@ static double* buildMappingPartId(apf::Mesh* m) int n = m->count(0); /* malloc instead of new[] for consistency with ph_read_field */ double* data = (double*)malloc(sizeof(double) * n); - int self = PCU_Comm_Self(); + int self = m->getPCU()->Self(); for (int i = 0; i < n; ++i) data[i] = self; return data; @@ -458,33 +457,35 @@ static double* buildMappingVtxId(apf::Mesh* m) return data; } -static std::string buildRestartFileName(std::string prefix, int step) +static std::string buildRestartFileName(std::string prefix, int step, pcu::PCU *PCUObj) { std::stringstream ss; - int rank = PCU_Comm_Self() + 1; + int rank = PCUObj->Self() + 1; ss << prefix << '.' << step << '.' << rank; return ss.str(); } void readAndAttachFields(Input& in, apf::Mesh* m) { - phastaio_initStats(); - double t0 = PCU_Time(); - setupInputSubdir(in.restartFileName); - std::string filename = buildRestartFileName(in.restartFileName, in.timeStepNumber); + PCU_t h; + h.ptr = static_cast(m->getPCU()); + phastaio_initStats(h); + double t0 = pcu::Time(); + setupInputSubdir(in.restartFileName, m->getPCU()); + std::string filename = buildRestartFileName(in.restartFileName, in.timeStepNumber, m->getPCU()); phastaio_setfile(RESTART_READ); - FILE* f = in.openfile_read(in, filename.c_str()); + FILE* f = in.openfile_read(in, filename.c_str(), m->getPCU()); if (!f) { lion_eprint(1,"failed to open \"%s\"!\n", filename.c_str()); abort(); } - int swap = ph_should_swap(f); + int swap = ph_should_swap(f, m->getPCU()->GetCHandle()); /* stops when ph_read_field returns 0 */ while( readAndAttachField(in,f,m,swap) ) {} PHASTAIO_CLOSETIME(fclose(f);) - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) lion_oprint(1,"fields read and attached in %f seconds\n", t1 - t0); - if(in.printIOtime) phastaio_printStats(); + if(in.printIOtime) phastaio_printStats(h); } static void destroyIfExists(apf::Mesh* m, const char* name) @@ -517,8 +518,8 @@ void attachZeroSolution(Input& in, apf::Mesh* m) void detachAndWriteSolution(Input& in, Output& out, apf::Mesh* m, std::string path) { - double t0 = PCU_Time(); - path += buildRestartFileName("restart", in.timeStepNumber); + double t0 = pcu::Time(); + path += buildRestartFileName("restart", in.timeStepNumber, m->getPCU()); phastaio_setfile(RESTART_WRITE); FILE* f = out.openfile_write(out, path.c_str()); if (!f) { @@ -564,8 +565,8 @@ void detachAndWriteSolution(Input& in, Output& out, apf::Mesh* m, std::string pa while(m->countFields()) apf::destroyField( m->getField(0) ); PHASTAIO_CLOSETIME(fclose(f);) - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!m->getPCU()->Self()) lion_oprint(1,"solution written in %f seconds\n", t1 - t0); } diff --git a/phasta/ph_convert.cc b/phasta/ph_convert.cc index 52e22a6af..880d5d7c9 100644 --- a/phasta/ph_convert.cc +++ b/phasta/ph_convert.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -20,7 +19,6 @@ #include #include #include - #include @@ -29,7 +27,7 @@ static void attachOrder(apf::Mesh* m) apf::numberOverlapDimension(m, "sim_order", m->getDimension()); } namespace { - static FILE* openFileRead(ph::Input&, const char* path) { + static FILE* openFileRead(ph::Input&, const char* path, pcu::PCU*) { return fopen(path, "r"); } @@ -68,7 +66,7 @@ int should_fix_pyramids = 1; int should_attach_order = 0; bool found_bad_arg = false; -void getConfig(int argc, char** argv) { +void getConfig(int argc, char** argv, pcu::PCU *pcu_obj) { opterr = 0; @@ -105,18 +103,18 @@ void getConfig(int argc, char** argv) { gmi_native_path = optarg; break; case '?': - if (!PCU_Comm_Self()) + if (!pcu_obj->Self()) lion_oprint(1,"warning: skipping unrecognized option\n"); break; default: - if (!PCU_Comm_Self()) + if (!pcu_obj->Self()) lion_oprint(1,"Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } } if(argc-optind != 3) { - if (!PCU_Comm_Self()) + if (!pcu_obj->Self()) lion_oprint(1,"Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } @@ -125,7 +123,7 @@ void getConfig(int argc, char** argv) { sms_path = argv[i++]; smb_path = argv[i++]; - if (!PCU_Comm_Self()) { + if (!pcu_obj->Self()) { lion_oprint(1,"fix_pyramids %d attach_order %d enable_log %d\n", should_fix_pyramids, should_attach_order, should_log); lion_oprint(1,"native-model \'%s\' model \'%s\' simmetrix mesh \'%s\' output mesh \'%s\'\n", @@ -136,7 +134,7 @@ void getConfig(int argc, char** argv) { static void fixCoords(apf::Mesh2* m) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); apf::MeshIterator* it = m->begin(0); apf::MeshEntity* e; apf::Vector3 x; @@ -148,24 +146,24 @@ static void fixCoords(apf::Mesh2* m) m->getPoint(e, 0, x); m->getParam(e, p); APF_ITERATE(apf::Copies, remotes, rit) { - PCU_COMM_PACK(rit->first, rit->second); - PCU_COMM_PACK(rit->first, x); - PCU_COMM_PACK(rit->first, p); + m->getPCU()->Pack(rit->first, rit->second); + m->getPCU()->Pack(rit->first, x); + m->getPCU()->Pack(rit->first, p); } } m->end(it); - PCU_Comm_Send(); + m->getPCU()->Send(); double max_x_diff = 0; double max_p_diff = 0; apf::Vector3 max_x_diff_point; apf::Vector3 max_p_diff_point; int x_diffs = 0; int p_diffs = 0; - while (PCU_Comm_Receive()) { + while (m->getPCU()->Receive()) { apf::Vector3 ox, op; - PCU_COMM_UNPACK(e); - PCU_COMM_UNPACK(ox); - PCU_COMM_UNPACK(op); + m->getPCU()->Unpack(e); + m->getPCU()->Unpack(ox); + m->getPCU()->Unpack(op); m->getPoint(e, 0, x); m->getParam(e, p); if (!(apf::areClose(p, op, 0.0))) { @@ -190,11 +188,11 @@ static void fixCoords(apf::Mesh2* m) double global_max[2]; global_max[0] = max_x_diff; global_max[1] = max_p_diff; - PCU_Max_Doubles(global_max, 2); + m->getPCU()->Max(global_max, 2); long global_diffs[2]; global_diffs[0] = x_diffs; global_diffs[1] = p_diffs; - PCU_Add_Longs(global_diffs, 2); + m->getPCU()->Add(global_diffs, 2); /* admittedly not the best way of checking which processor had the max */ if (global_diffs[0] && (global_max[0] == max_x_diff)) @@ -216,18 +214,19 @@ static void postConvert(apf::Mesh2* m) int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); MS_init(); SimModel_start(); Sim_readLicenseFile(NULL); SimPartitionedMesh_start(&argc,&argv); - getConfig(argc, argv); + getConfig(argc, argv, &pcu_obj); if( should_log ) Sim_logOn("convert.sim.log"); if (should_attach_order && should_fix_pyramids) { - if (!PCU_Comm_Self()) + if (!pcu_obj.Self()) std::cout << "disabling pyramid fix because --attach-order was given\n"; should_fix_pyramids = false; } @@ -243,21 +242,21 @@ int main(int argc, char** argv) else mdl = gmi_load(gmi_path); pGModel simModel = gmi_export_sim(mdl); - double t0 = PCU_Time(); + double t0 = pcu::Time(); pParMesh sim_mesh = PM_load(sms_path, simModel, progress); - double t1 = PCU_Time(); - if(!PCU_Comm_Self()) + double t1 = pcu::Time(); + if(!pcu_obj.Self()) lion_eprint(1, "read and created the simmetrix mesh in %f seconds\n", t1-t0); - apf::Mesh* simApfMesh = apf::createMesh(sim_mesh); - double t2 = PCU_Time(); - if(!PCU_Comm_Self()) + apf::Mesh* simApfMesh = apf::createMesh(sim_mesh, &pcu_obj); + double t2 = pcu::Time(); + if(!simApfMesh->getPCU()->Self()) lion_eprint(1, "created the apf_sim mesh in %f seconds\n", t2-t1); if (should_attach_order) attachOrder(simApfMesh); ph::buildMapping(simApfMesh); apf::Mesh2* mesh = apf::createMdsMesh(mdl, simApfMesh); - double t3 = PCU_Time(); - if(!PCU_Comm_Self()) + double t3 = pcu::Time(); + if(!mesh->getPCU()->Self()) lion_eprint(1, "created the apf_mds mesh in %f seconds\n", t3-t2); apf::printStats(mesh); @@ -265,7 +264,7 @@ int main(int argc, char** argv) M_release(sim_mesh); postConvert(mesh); mesh->writeNative(smb_path); - std::string restartPath = ph::setupOutputDir(); + std::string restartPath = ph::setupOutputDir(mesh->getPCU()); ph::Input phIn; phIn.openfile_read = openFileRead; ph::Output phOut; @@ -290,6 +289,6 @@ int main(int argc, char** argv) MS_exit(); if( should_log ) Sim_logOff(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/phiotimer.cc b/phasta/phiotimer.cc index 6e1a9502c..b4b4ffc0d 100644 --- a/phasta/phiotimer.cc +++ b/phasta/phiotimer.cc @@ -1,7 +1,8 @@ #include #include -#include +#include #include +#include #include #include @@ -32,7 +33,7 @@ void phastaio_time(phastaioTime* t) { *t = _rdtsc(); //intel intrinsic } /* determine the reference clock frequency */ -static size_t phastaio_getCyclesPerMicroSec() { +static size_t phastaio_getCyclesPerMicroSec(PCU_t h) { const size_t usec = 5*MILLION; size_t cpus, cycles; phastaioTime t0, t1; @@ -44,7 +45,7 @@ static size_t phastaio_getCyclesPerMicroSec() { phastaio_time(&t1); cycles = t1 - t0; cpus = ((double)cycles)/(usec); - if(!PCU_Comm_Self()) { + if(!PCU_Comm_Self(h)) { std::stringstream ss; ss << "cycles " << cycles << " us " << usec << " cycles per micro second " << cpus << "\n"; @@ -61,7 +62,7 @@ size_t phastaio_time_diff(phastaioTime* start, phastaioTime* end) { } #elif defined(USE_PCU_TIME) void phastaio_time(phastaioTime* t) { - *t = PCU_Time(); + *t = pcu::Time(); } /*return elapsed time in micro seconds*/ size_t phastaio_time_diff(phastaioTime* start, phastaioTime* end) { @@ -140,12 +141,12 @@ static const char* getFileName() { return names[phastaio_global_stats.fileIdx]; } -static void printMinMaxAvgSzt(const char* key, size_t v) { - size_t min = PCU_Min_SizeT(v); - size_t max = PCU_Max_SizeT(v); - size_t tot = PCU_Add_SizeT(v); - double avg = ((double)tot)/PCU_Comm_Peers(); - if(!PCU_Comm_Self()) { +static void printMinMaxAvgSzt(const char* key, size_t v, PCU_t h) { + size_t min = PCU_Min_SizeT(h, v); + size_t max = PCU_Max_SizeT(h, v); + size_t tot = PCU_Add_SizeT(h, v); + double avg = ((double)tot)/PCU_Comm_Peers(h); + if(!PCU_Comm_Self(h)) { std::stringstream ss; ss << getFileName() << "_" << key << "min max avg" << min << " " << max << " " << avg << "\n"; @@ -154,12 +155,12 @@ static void printMinMaxAvgSzt(const char* key, size_t v) { } } -static void printMinMaxAvgDbl(const char* key, double v) { - double min = PCU_Min_Double(v); - double max = PCU_Max_Double(v); - double tot = PCU_Add_Double(v); - double avg = tot/PCU_Comm_Peers(); - if(!PCU_Comm_Self()) +static void printMinMaxAvgDbl(const char* key, double v, PCU_t h) { + double min = PCU_Min_Double(h, v); + double max = PCU_Max_Double(h, v); + double tot = PCU_Add_Double(h, v); + double avg = tot/PCU_Comm_Peers(h); + if(!PCU_Comm_Self(h)) lion_eprint(1, "%s_%s min max avg %f %f %f\n", getFileName(), key, min, max, avg); } @@ -214,8 +215,8 @@ static size_t phastaio_getCloseTime() { return phastaio_global_stats.closeTime[i]; } -void phastaio_printStats() { - if(!PCU_Comm_Self()) { +void phastaio_printStats(PCU_t h) { + if(!PCU_Comm_Self(h)) { const size_t us = 1000; phastaioTime t0,t1; size_t elapsed; @@ -232,58 +233,57 @@ void phastaio_printStats() { size_t totalus = 0; size_t totalbytes = 0; phastaio_setfile(chefFile); - if(!PCU_Comm_Self()) + if(!PCU_Comm_Self(h)) lion_eprint(1, "phastaio_filename %s\n", getFileName()); - int reads = PCU_Max_Int((int)phastaio_getReads()); + int reads = PCU_Max_Int(h, (int)phastaio_getReads()); if(reads) { totalus += phastaio_getReadTime(); totalbytes += phastaio_getReadBytes(); - printMinMaxAvgSzt("reads", phastaio_getReads()); - printMinMaxAvgSzt("readTime (us)", phastaio_getReadTime()); - printMinMaxAvgSzt("readBytes (B)", phastaio_getReadBytes()); + printMinMaxAvgSzt("reads", phastaio_getReads(), h); + printMinMaxAvgSzt("readTime (us)", phastaio_getReadTime(), h); + printMinMaxAvgSzt("readBytes (B)", phastaio_getReadBytes(), h); double bw = ((double)phastaio_getReadBytes())/phastaio_getReadTime(); - printMinMaxAvgDbl("readBandwidth (MB/s)", bw); + printMinMaxAvgDbl("readBandwidth (MB/s)", bw, h); /* B * 10^6us * 1MB = MB * - ------ ----- -- * us 1s 10^6B s */ } - int writes = PCU_Max_Int((int)phastaio_getWrites()); + int writes = PCU_Max_Int(h, (int)phastaio_getWrites()); if(writes) { totalus += phastaio_getWriteTime(); totalbytes += phastaio_getWriteBytes(); - printMinMaxAvgSzt("writes", phastaio_getWrites()); - printMinMaxAvgSzt("writeTime (us)", phastaio_getWriteTime()); - printMinMaxAvgSzt("writeBytes (B)", phastaio_getWriteBytes()); + printMinMaxAvgSzt("writes", phastaio_getWrites(), h); + printMinMaxAvgSzt("writeTime (us)", phastaio_getWriteTime(), h); + printMinMaxAvgSzt("writeBytes (B)", phastaio_getWriteBytes(), h); printMinMaxAvgDbl("writeBandwidth (MB/s)", - ((double)phastaio_getWriteBytes())/phastaio_getWriteTime()); + ((double)phastaio_getWriteBytes())/phastaio_getWriteTime(), h); } - int opens = PCU_Max_Int((int)phastaio_getOpens()); + int opens = PCU_Max_Int(h, (int)phastaio_getOpens()); if(opens) { totalus += phastaio_getOpenTime(); - printMinMaxAvgSzt("opens", phastaio_getOpens()); - printMinMaxAvgSzt("openTime (us)", phastaio_getOpenTime()); + printMinMaxAvgSzt("opens", phastaio_getOpens(), h); + printMinMaxAvgSzt("openTime (us)", phastaio_getOpenTime(), h); } - int closes = PCU_Max_Int((int)phastaio_getCloses()); + int closes = PCU_Max_Int(h, (int)phastaio_getCloses()); if(closes) { totalus += phastaio_getCloseTime(); - printMinMaxAvgSzt("closes", phastaio_getCloses()); - printMinMaxAvgSzt("closeTime (us)", phastaio_getCloseTime()); + printMinMaxAvgSzt("closes", phastaio_getCloses(), h); + printMinMaxAvgSzt("closeTime (us)", phastaio_getCloseTime(), h); } if(totalbytes) { - printMinMaxAvgSzt("totalTime (us)", totalus); - printMinMaxAvgSzt("totalBytes (B)", totalbytes); + printMinMaxAvgSzt("totalTime (us)", totalus, h); + printMinMaxAvgSzt("totalBytes (B)", totalbytes, h); printMinMaxAvgDbl("effectiveBandwidth (MB/s)", - ((double)totalbytes)/totalus); + ((double)totalbytes)/totalus, h); } } } -void phastaio_initStats() { - if( !PCU_Comm_Initialized() ) - PCU_Comm_Init(); +void phastaio_initStats(PCU_t h) { + (void) h; #ifdef __INTEL_COMPILER - phastaio_global_stats.cpus = phastaio_getCyclesPerMicroSec(); + phastaio_global_stats.cpus = phastaio_getCyclesPerMicroSec(h); #endif for(int i=0; iGetMPIComm(), &rank); return !rank; } #endif - inline void printTime(const char* key, double t) { + inline void printTime(const char* key, double t, pcu::PCU *pcu_obj) { (void) key; (void) t; + (void) pcu_obj; #if PHSTREAM_TIMERS_ON==1 - if( isRankZero() ) + if( isRankZero(pcu_obj) ) lion_eprint(1, "%s %f seconds\n", key, t); #endif } @@ -44,80 +45,80 @@ extern "C" { }; } -RStream* makeRStream() { +RStream* makeRStream(pcu::PCU *PCUObj) { const double t0 = getTime(); RStream* rs = (RStream*) malloc(sizeof(RStream)); rs->restart = NULL; rs->rSz = 0; - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); return rs; } #ifdef __APPLE__ -FILE* openRStreamRead(RStream*) { +FILE* openRStreamRead(RStream*, pcu::PCU*) { return NULL; } #else -FILE* openRStreamRead(RStream* rs) { +FILE* openRStreamRead(RStream* rs, pcu::PCU *PCUObj) { const double t0 = getTime(); FILE* f = fmemopen(rs->restart, rs->rSz, "r"); - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); return f; } #endif #ifdef __APPLE__ -FILE* openRStreamWrite(RStream*) { +FILE* openRStreamWrite(RStream*, pcu::PCU*) { return NULL; } #else -FILE* openRStreamWrite(RStream* rs) { +FILE* openRStreamWrite(RStream* rs, pcu::PCU *PCUObj) { const double t0 = getTime(); FILE* f = open_memstream(&(rs->restart), &(rs->rSz)); - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); return f; } #endif -void clearRStream(RStream* rs) { +void clearRStream(RStream* rs, pcu::PCU *PCUObj) { const double t0 = getTime(); if(rs->restart) { free(rs->restart); rs->restart = NULL; rs->rSz = 0; } - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); } -void destroyRStream(RStream* rs) { +void destroyRStream(RStream* rs, pcu::PCU *PCUObj) { const double t0 = getTime(); - clearRStream(rs); + clearRStream(rs, PCUObj); free(rs); - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); } -void attachRStream(GRStream* grs, RStream* rs) { +void attachRStream(GRStream* grs, RStream* rs, pcu::PCU *PCUObj) { const double t0 = getTime(); rs->restart = grs->restart; rs->rSz = grs->rSz; grs->restart = NULL; grs->rSz = 0; - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); } -GRStream* makeGRStream() { +GRStream* makeGRStream(pcu::PCU *PCUObj) { const double t0 = getTime(); GRStream* grs = (GRStream*) malloc(sizeof(GRStream)); grs->geom = NULL; grs->gSz = 0; grs->restart = NULL; grs->rSz = 0; - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); return grs; } -void whichStream(const char* name, bool& isR, bool& isG) { +void whichStream(const char* name, bool& isR, bool& isG, pcu::PCU *PCUObj) { const double t0 = getTime(); std::string fname(name); std::string restartStr("restart"); @@ -125,7 +126,7 @@ void whichStream(const char* name, bool& isR, bool& isG) { isR = (fname.find(restartStr) != std::string::npos); isG = (fname.find(geombcStr) != std::string::npos); PCU_ALWAYS_ASSERT(isR != isG); - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); } void writeUnknown(const char* fname) { @@ -135,14 +136,14 @@ void writeUnknown(const char* fname) { } #ifdef __APPLE__ -FILE* openGRStreamRead(GRStream*, const char*) { +FILE* openGRStreamRead(GRStream*, const char*, pcu::PCU*) { return NULL; } #else -FILE* openGRStreamRead(GRStream* grs, const char* named) { +FILE* openGRStreamRead(GRStream* grs, const char* named, pcu::PCU *PCUObj) { const double t0 = getTime(); bool isR, isG; - whichStream(named, isR, isG); + whichStream(named, isR, isG, PCUObj); FILE* f = NULL; if( isR && !isG ) f = fmemopen(grs->restart, grs->rSz, "r"); @@ -152,20 +153,20 @@ FILE* openGRStreamRead(GRStream* grs, const char* named) { writeUnknown(named); exit(1); } - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); return f; } #endif #ifdef __APPLE__ -FILE* openGRStreamWrite(GRStream*, const char*) { +FILE* openGRStreamWrite(GRStream*, const char*, pcu::PCU*) { return NULL; } #else -FILE* openGRStreamWrite(GRStream* grs, const char* named) { +FILE* openGRStreamWrite(GRStream* grs, const char* named, pcu::PCU *PCUObj) { const double t0 = getTime(); bool isR, isG; - whichStream(named, isR, isG); + whichStream(named, isR, isG, PCUObj); FILE* f = NULL; if( isR && !isG ) f = open_memstream(&(grs->restart), &(grs->rSz)); @@ -175,12 +176,12 @@ FILE* openGRStreamWrite(GRStream* grs, const char* named) { writeUnknown(named); exit(1); } - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); return f; } #endif -void clearGRStream(GRStream* grs) { +void clearGRStream(GRStream* grs, pcu::PCU *PCUObj) { const double t0 = getTime(); if(grs->geom) { free(grs->geom); @@ -192,13 +193,13 @@ void clearGRStream(GRStream* grs) { grs->restart = NULL; grs->rSz = 0; } - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); } -void destroyGRStream(GRStream* grs) { +void destroyGRStream(GRStream* grs, pcu::PCU *PCUObj) { const double t0 = getTime(); - clearGRStream(grs); + clearGRStream(grs, PCUObj); free(grs); - printTime(__func__, getTime()-t0); + printTime(__func__, getTime()-t0, PCUObj); } diff --git a/phasta/phstream.h b/phasta/phstream.h index 6a039e89c..316e074c5 100644 --- a/phasta/phstream.h +++ b/phasta/phstream.h @@ -1,6 +1,7 @@ #ifndef PHSTREAM_H_ #define PHSTREAM_H_ #include +#include /** \file phstream.h @@ -13,29 +14,29 @@ typedef struct RStream* rstream; typedef struct GRStream* grstream; /** @brief make restart stream */ -rstream makeRStream(); +rstream makeRStream(pcu::PCU *PCUObj); /** @brief clear restart stream */ -void clearRStream(rstream rs); +void clearRStream(rstream rs, pcu::PCU *PCUObj); /** @brief detach output stream */ -void destroyRStream(rstream rs); +void destroyRStream(rstream rs, pcu::PCU *PCUObj); /** @brief make geom-restart stream */ -grstream makeGRStream(); +grstream makeGRStream(pcu::PCU *PCUObj); /** @brief clear geom-restart stream */ -void clearGRStream(grstream grs); +void clearGRStream(grstream grs, pcu::PCU *PCUObj); /** @brief destroy geom-restart stream */ -void destroyGRStream(grstream grs); +void destroyGRStream(grstream grs, pcu::PCU *PCUObj); /** @brief open restart stream for reading*/ -FILE* openRStreamRead(rstream rs); +FILE* openRStreamRead(rstream rs, pcu::PCU *PCUObj); /** @brief open restart stream for writing*/ -FILE* openRStreamWrite(rstream rs); +FILE* openRStreamWrite(rstream rs, pcu::PCU *PCUObj); /** @brief open named stream in geom-restart stream for reading*/ -FILE* openGRStreamRead(grstream grs, const char* named); +FILE* openGRStreamRead(grstream grs, const char* named, pcu::PCU *PCUObj); /** @brief open named stream in geom-restart stream for writing*/ -FILE* openGRStreamWrite(grstream grs, const char* named); +FILE* openGRStreamWrite(grstream grs, const char* named, pcu::PCU *PCUObj); /** @brief dev function */ -void attachRStream(grstream grs, rstream rs); +void attachRStream(grstream grs, rstream rs, pcu::PCU *PCUObj); #endif diff --git a/phasta/readUrPrep.cc b/phasta/readUrPrep.cc index 01c57ce1d..71eba7065 100644 --- a/phasta/readUrPrep.cc +++ b/phasta/readUrPrep.cc @@ -10,7 +10,7 @@ namespace { m->destroyNative(); apf::destroyMesh(m); } - static FILE* openfile_read(ph::Input&, const char* path) { + static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU*) { return fopen(path, "r"); } } @@ -18,19 +18,20 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); - PCU_Protect(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pcu::Protect(); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2], &pcu_obj); ph::Input ctrl; - ctrl.load(argv[3]); + ctrl.load(argv[3], &pcu_obj); ctrl.openfile_read = openfile_read; ctrl.buildMapping = 0; //can't map new vertices from UR chef::readAndAttachFields(ctrl,m); chef::uniformRefinement(ctrl,m); chef::preprocess(m,ctrl); freeMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/phasta/threshold.cc b/phasta/threshold.cc index 7da7ee2c4..b619bf0a9 100644 --- a/phasta/threshold.cc +++ b/phasta/threshold.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -20,7 +19,8 @@ static double process_element(apf::Vector3 x[4], double sol[4][9]) int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); gmi_sim_start(); @@ -29,7 +29,7 @@ int main(int argc, char** argv) gmi_register_mesh(); lion_set_verbosity(1); ph::Input in; - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], &pcu_obj); m->verify(); in.restartFileName = argv[3]; in.timeStepNumber = 0; @@ -58,6 +58,6 @@ int main(int argc, char** argv) gmi_sim_stop(); Sim_unregisterAllKeys(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/pumi/pumi.h b/pumi/pumi.h index 529d528ba..3a3cd91fc 100644 --- a/pumi/pumi.h +++ b/pumi/pumi.h @@ -15,6 +15,7 @@ #include "GenIterator.h" #include "mPartEntityContainer.h" #include "apf.h" +#include "pcu_util.h" enum PUMI_EntTopology { PUMI_VERTEX, // 0 @@ -90,6 +91,11 @@ class pumi static pumi _instance; return &_instance; }; + void initializePCU(pcu::PCU *newPCU) { + PCU_ALWAYS_ASSERT_VERBOSE(PCUObj==nullptr, "pumi::instance() PCUObj already initialized\n"); + PCUObj = newPCU; + } + [[nodiscard]] pcu::PCU* getPCU() const noexcept {return PCUObj;} pMesh mesh; pGeom model; @@ -100,6 +106,9 @@ class pumi pMeshTag ghost_tag; std::vector ghost_vec[4]; std::vector ghosted_vec[4]; + + private: + pcu::PCU *PCUObj; }; //************************************ @@ -107,13 +116,12 @@ class pumi // 0- SYSTEM-LEVEL FUNCTIONS //************************************ //************************************ -void pumi_start(); -void pumi_finalize(bool do_mpi_finalize=true); +void pumi_load_pcu(pcu::PCU *PCUObj); int pumi_size(); int pumi_rank(); -void pumi_sync(void); +void pumi_sync(); void pumi_printSys(); double pumi_getTime(); double pumi_getMem(); diff --git a/pumi/pumi_field.cc b/pumi/pumi_field.cc index 74fb55574..77d0e0398 100644 --- a/pumi/pumi_field.cc +++ b/pumi/pumi_field.cc @@ -389,7 +389,7 @@ static void sendFieldData(pMesh m, pMeshEnt e, pField f, int nf, pOwnership shr) double *s_data = (double*)((char*)msg_send+sizeof(pMeshEnt)+sizeof(int)); for (int pos=0; posgetPCU()->Write(to, (void*)msg_send, msg_size); free(msg_send); } @@ -408,13 +408,13 @@ static void sendFieldData(pMesh m, pMeshEnt e, pField f, int nf, pOwnership shr) double *s_data = (double*)((char*)msg_send+sizeof(pMeshEnt)+sizeof(int)); for (int pos=0; posgetPCU()->Write(to, (void*)msg_send, msg_size); free(msg_send); } } //if (m->isGhosted(e)) } -static void receiveFieldData(std::vector& fields, std::set& mismatch_fields) +static void receiveFieldData(std::vector& fields, std::set& mismatch_fields, pcu::PCU *PCUObj) { pField f; void *msg_recv; @@ -422,7 +422,7 @@ static void receiveFieldData(std::vector& fields, std::set& mism size_t msg_size; pMeshEnt e; - while(PCU_Comm_Read(&pid_from, &msg_recv, &msg_size)) + while(PCUObj->Read(&pid_from, &msg_recv, &msg_size)) { e = *((pMeshEnt*)msg_recv); int *nf = (int*)((char*)msg_recv+sizeof(pMeshEnt)); @@ -484,26 +484,26 @@ void pumi_field_verify(pMesh m, pField f, pOwnership shr) if (!static_cast(fields[nf])->getShape()->hasNodesIn(d)) continue; - PCU_Comm_Begin(); + m->getPCU()->Begin(); pMeshIter it = m->begin(d); pMeshEnt e; while ((e = m->iterate(it))) sendFieldData(m, e, fields[nf], nf, shr); m->end(it); - PCU_Comm_Send(); - receiveFieldData(fields,mismatch_fields); + m->getPCU()->Send(); + receiveFieldData(fields,mismatch_fields,m->getPCU()); } } - int global_size = PCU_Max_Int((int)mismatch_fields.size()); + int global_size = m->getPCU()->Max((int)mismatch_fields.size()); if (global_size) { - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) for (std::set::iterator it=mismatch_fields.begin(); it!=mismatch_fields.end(); ++it) lion_oprint(1,"%s: \"%s\" DOF mismatch over remote/ghost copies\n", __func__, getName(*it)); } else { - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) lion_oprint(1,"%s: no DOF mismatch\n", __func__); } } diff --git a/pumi/pumi_gentity.cc b/pumi/pumi_gentity.cc index 28a6df473..eabf504e6 100644 --- a/pumi/pumi_gentity.cc +++ b/pumi/pumi_gentity.cc @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/pumi/pumi_geom.cc b/pumi/pumi_geom.cc index 6fe0edb5b..2305959b1 100644 --- a/pumi/pumi_geom.cc +++ b/pumi/pumi_geom.cc @@ -12,7 +12,7 @@ #include "gmi_null.h" #include "gmi_analytic.h" #include "pumi_iter.h" -#include "PCU.h" +#include #include #include #include @@ -54,7 +54,7 @@ pGeom pumi_geom_load(const char* filename, const char* model_type, void (*geom_l pGeom pumi_geom_load(gmi_model* gm, const char* model_type, const char* filename, void (*geom_load_fp)(const char*)) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); if (!strcmp(model_type,"null")) pumi::instance()->model = new gModel(gm); else if (!strcmp(model_type,"mesh")) @@ -77,8 +77,8 @@ pGeom pumi_geom_load(gmi_model* gm, const char* model_type, return NULL; } - if (!PCU_Comm_Self() && filename) - lion_oprint(1,"model %s loaded in %f seconds\n", filename, PCU_Time() - t0); + if (!pumi_rank() && filename) + lion_oprint(1,"model %s loaded in %f seconds\n", filename, pcu::Time() - t0); return pumi::instance()->model; } @@ -193,7 +193,7 @@ void pumi_giter_reset(gIter iter) void pumi_geom_print (pGeom g, bool print_ent) { - if (PCU_Comm_Self()) return; + if (pumi_rank()) return; std::cout<<"\n=== model entity and tag info === \n"; std::cout<<"# global geom ent: v "<size(0)<<", e " <size(1)<<", f "<size(2)<<", r "<size(3)<<"\n"; diff --git a/pumi/pumi_ghost.cc b/pumi/pumi_ghost.cc index e8dfdacdd..0a6235b76 100644 --- a/pumi/pumi_ghost.cc +++ b/pumi/pumi_ghost.cc @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -84,7 +83,7 @@ void Ghosting::send(pMeshEnt e, int to) /** assign a destination part id of all entities of dimension */ void Ghosting::send (int to) { - if (to==PCU_Comm_Self()) return; + if (to==m->getPCU()->Self()) return; pMeshEnt e; apf::MeshIterator* it = m->begin(ghost_dim); @@ -104,7 +103,7 @@ void Ghosting::print() m->getIntTag(e, parts_index_tag, &index); APF_ITERATE(Parts,*(parts_vec[ghost_dim][index]),pit) - std::cout<<"("<getPCU()->Self()<<") ghost e "<end(it); } @@ -135,9 +134,9 @@ int Ghosting::count() static pMeshEnt unpackGhost(Ghosting* plan, apf::DynamicArray& tags) // ********************************************************* { - int from = PCU_Comm_Sender(); + int from = plan->getMesh()->getPCU()->Sender(); int type; - PCU_COMM_UNPACK(type); + plan->getMesh()->getPCU()->Unpack(type); pMeshEnt sender; apf::ModelEntity* c; Parts residence; @@ -165,7 +164,7 @@ static void ghost_receiveEntities(Ghosting* plan, apf::DynamicArray& t // ********************************************************* { received.reserve(1024); - while (PCU_Comm_Receive()) + while (plan->getMesh()->getPCU()->Receive()) received.push_back(unpackGhost(plan,tags)); } @@ -173,7 +172,7 @@ static void ghost_receiveEntities(Ghosting* plan, apf::DynamicArray& t static void setupGhosts(pMesh m, EntityVector& received) // ********************************************************* { - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(EntityVector,received,it) { pMeshEnt entity = *it; @@ -182,24 +181,24 @@ static void setupGhosts(pMesh m, EntityVector& received) apf::Copies temp; m->getGhosts(entity,temp); int to = temp.begin()->first; - PCU_COMM_PACK(to,temp.begin()->second); // sender - PCU_COMM_PACK(to,entity); + m->getPCU()->Pack(to,temp.begin()->second); // sender + m->getPCU()->Pack(to,entity); apf::Copies remotes; m->getRemotes(entity, remotes); APF_ITERATE(Copies,remotes,rit) { - PCU_COMM_PACK(rit->first, rit->second); // sender - PCU_COMM_PACK(rit->first,entity); + m->getPCU()->Pack(rit->first, rit->second); // sender + m->getPCU()->Pack(rit->first,entity); } } - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { - int from = PCU_Comm_Sender(); + int from = m->getPCU()->Sender(); pMeshEnt entity; - PCU_COMM_UNPACK(entity); + m->getPCU()->Unpack(entity); pMeshEnt sender; - PCU_COMM_UNPACK(sender); + m->getPCU()->Unpack(sender); m->addGhost(entity, from, sender); if (!m->hasTag(entity, pumi::instance()->ghosted_tag)) { @@ -265,7 +264,7 @@ static void ghost_collectEntities (pMesh m, Ghosting* plan, EntityVector entitie size_t msg_size; for (int dim = 0; dim <=ghost_dim; ++dim) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(EntityVector,entitiesToGhost[dim],it) { e = *it; @@ -289,18 +288,18 @@ static void ghost_collectEntities (pMesh m, Ghosting* plan, EntityVector entitie pids[pos]=*pit; ++pos; } - PCU_Comm_Write(rit->first, (void*)msg_send, msg_size); + m->getPCU()->Write(rit->first, (void*)msg_send, msg_size); free(msg_send); } } // for entitiesToGhost[dim] - PCU_Comm_Send(); + m->getPCU()->Send(); // receive phase void *msg_recv; int pid_from; int* pids; pMeshEnt r; - while(PCU_Comm_Read(&pid_from, &msg_recv, &msg_size)) + while(m->getPCU()->Read(&pid_from, &msg_recv, &msg_size)) { r = *((pMeshEnt*)msg_recv); if ( !m->hasTag(r,tag)) @@ -341,7 +340,7 @@ void ghost_sendEntities(Ghosting* plan, int entDim, // ********************************************** { pMeshEnt ent; - int src_partid=PCU_Comm_Self(); + int src_partid=plan->getMesh()->getPCU()->Self(); pMesh m = plan->getMesh(); std::set res_parts, temp; @@ -394,7 +393,7 @@ void ghost_sendEntities(Ghosting* plan, int entDim, void pumi_ghost_create(pMesh m, Ghosting* plan) // ********************************************************* { - if (PCU_Comm_Peers()==1) { + if (m->getPCU()->Peers()==1) { delete plan; return; } @@ -411,7 +410,7 @@ void pumi_ghost_create(pMesh m, Ghosting* plan) } } - double t0=PCU_Time(); + double t0=pcu::Time(); EntityVector entities_to_ghost[4]; ghost_collectEntities(m, plan, entities_to_ghost); @@ -420,9 +419,9 @@ void pumi_ghost_create(pMesh m, Ghosting* plan) plan->getMesh()->getTags(tags); for (int dimension = 0; dimension <= plan->ghost_dim; ++dimension) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); ghost_sendEntities(plan, dimension, entities_to_ghost[dimension], tags); - PCU_Comm_Send(); + m->getPCU()->Send(); EntityVector received; ghost_receiveEntities(plan,tags,received); setupGhosts(plan->getMesh(),received); @@ -439,8 +438,8 @@ void pumi_ghost_create(pMesh m, Ghosting* plan) for (std::vector::iterator fit=frozen_fields.begin(); fit!=frozen_fields.end(); ++fit) apf::freeze(*fit); - if (!PCU_Comm_Self()) - lion_oprint(1,"mesh ghosted in %f seconds\n", PCU_Time()-t0); + if (!m->getPCU()->Self()) + lion_oprint(1,"mesh ghosted in %f seconds\n", pcu::Time()-t0); } // ********************************************************* @@ -455,7 +454,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, pMeshEnt* s_ent; size_t msg_size; int dummy=1; - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int layer=2; layerfirst, (void*)msg_send, msg_size); + m->getPCU()->Write(brg_rit->first, (void*)msg_send, msg_size); free(msg_send); } // APF_ITERATE } // for (std::map >::iterator iter @@ -488,7 +487,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int j=0; jgetPCU()->Send(); // receive phase void *msg_recv; int pid_from; @@ -499,7 +498,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, std::vector processed_ent; std::vector adj_ent; - while(PCU_Comm_Read(&pid_from, &msg_recv, &msg_size)) + while(m->getPCU()->Read(&pid_from, &msg_recv, &msg_size)) { processed_ent.clear(); @@ -579,11 +578,11 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int i=0; igetPCU()->GetMPIComm()); while (global_num_off_part) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); for (int layer=0; layerfirst, (void*)msg_send, msg_size); + m->getPCU()->Write(brg_rit->first, (void*)msg_send, msg_size); free(msg_send); } // APF_ITERATE } // for off_it @@ -616,9 +615,9 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int j=0; jgetPCU()->Send(); - while (PCU_Comm_Read(&pid_from, &msg_recv, &msg_size)) + while (m->getPCU()->Read(&pid_from, &msg_recv, &msg_size)) { processed_ent.clear(); @@ -700,7 +699,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int i=0; igetPCU()->GetMPIComm()); } // while global_off_part_brg } @@ -709,7 +708,7 @@ void do_off_part_bridge(pMesh m, int brg_dim, int ghost_dim, int num_layer, void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, int include_copy) // ********************************************************* { - if (PCU_Comm_Peers()==1 || num_layer==0) return; + if (m->getPCU()->Peers()==1 || num_layer==0) return; int dummy=1, mesh_dim=m->getDimension(), self = pumi_rank();; @@ -722,7 +721,7 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, return; } - double t0 = PCU_Time(); + double t0 = pcu::Time(); pMeshTag tag = m->createIntTag("ghost_check_mark",1); Ghosting* plan = new Ghosting(m, ghost_dim); @@ -824,7 +823,7 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, for (int i=0; igetPCU()->GetMPIComm()); } if (global_num_off_part) @@ -843,8 +842,8 @@ void pumi_ghost_createLayer (pMesh m, int brg_dim, int ghost_dim, int num_layer, // ******************************************** // STEP 3: perform ghosting // ******************************************** - if (!PCU_Comm_Self()) - lion_oprint(1,"ghosting plan computed in %f seconds\n", PCU_Time()-t0); + if (!m->getPCU()->Self()) + lion_oprint(1,"ghosting plan computed in %f seconds\n", pcu::Time()-t0); pumi_ghost_create(m, plan); } diff --git a/pumi/pumi_mentity.cc b/pumi/pumi_mentity.cc index 8a410657b..341a680ad 100644 --- a/pumi/pumi_mentity.cc +++ b/pumi/pumi_mentity.cc @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include diff --git a/pumi/pumi_mesh.cc b/pumi/pumi_mesh.cc index b48ed473f..587e87a65 100644 --- a/pumi/pumi_mesh.cc +++ b/pumi/pumi_mesh.cc @@ -30,7 +30,7 @@ using std::map; // mesh creation pMesh pumi_mesh_create(pGeom g, int mesh_dim, bool periodic) { - pumi::instance()->mesh = apf::makeEmptyMdsMesh(g->getGmi(), mesh_dim, periodic); + pumi::instance()->mesh = apf::makeEmptyMdsMesh(g->getGmi(), mesh_dim, periodic, pumi::instance()->getPCU()); return pumi::instance()->mesh; } @@ -69,10 +69,10 @@ void generate_globalid(pMesh m, pMeshTag tag, int dim, pOwnership o) ++num_own; m->end(it); - PCU_Exscan_Ints(&num_own,1); + m->getPCU()->Exscan(&num_own,1); int initial_id=num_own; - PCU_Comm_Begin(); + m->getPCU()->Begin(); it = m->begin(dim); while ((e = m->iterate(it))) { @@ -84,8 +84,8 @@ void generate_globalid(pMesh m, pMeshTag tag, int dim, pOwnership o) m->getRemotes(e, remotes); APF_ITERATE(Copies, remotes, it) { - PCU_COMM_PACK(it->first, it->second); - PCU_Comm_Pack(it->first, &initial_id, sizeof(int)); + m->getPCU()->Pack(it->first, it->second); + m->getPCU()->Pack(it->first, &initial_id, sizeof(int)); } if (m->isGhosted(e)) @@ -94,22 +94,22 @@ void generate_globalid(pMesh m, pMeshTag tag, int dim, pOwnership o) m->getGhosts(e, ghosts); APF_ITERATE(Copies, ghosts, it) { - PCU_COMM_PACK(it->first, it->second); - PCU_Comm_Pack(it->first, &initial_id, sizeof(int)); + m->getPCU()->Pack(it->first, it->second); + m->getPCU()->Pack(it->first, &initial_id, sizeof(int)); } } ++initial_id; } m->end(it); - PCU_Comm_Send(); + m->getPCU()->Send(); int global_id; - while (PCU_Comm_Listen()) - while (!PCU_Comm_Unpacked()) + while (m->getPCU()->Listen()) + while (!m->getPCU()->Unpacked()) { pMeshEnt remote_ent; - PCU_COMM_UNPACK(remote_ent); - PCU_Comm_Unpack(&global_id, sizeof(int)); + m->getPCU()->Unpack(remote_ent); + m->getPCU()->Unpack(&global_id, sizeof(int)); m->setIntTag(remote_ent, tag, &global_id); } } @@ -178,46 +178,49 @@ apf::Migration* getPlan(apf::Mesh* m, int num_target_part) return plan; } -void split_comm(int num_out_comm) +void split_comm(int num_out_comm, pcu::PCU &PCUObj) { - int self = PCU_Comm_Self(); + int self = PCUObj.Self(); int group_id = self % num_out_comm; int in_group_rank = self / num_out_comm; MPI_Comm groupComm; - MPI_Comm_split(PCU_Get_Comm(), group_id, in_group_rank, &groupComm); - PCU_Switch_Comm(groupComm); + MPI_Comm_split(PCUObj.GetMPIComm(), group_id, in_group_rank, &groupComm); + PCUObj.SwitchMPIComm(groupComm); } -void merge_comm(MPI_Comm oldComm) + +void merge_comm(MPI_Comm oldComm, pcu::PCU &PCUObj) { - MPI_Comm prevComm = PCU_Get_Comm(); - PCU_Switch_Comm(oldComm); + MPI_Comm prevComm = PCUObj.GetMPIComm(); + PCUObj.SwitchMPIComm(oldComm); MPI_Comm_free(&prevComm); } + pGeom pumi_mesh_getGeom(pMesh) { return pumi::instance()->model; } -// load a serial mesh on master process then distribute as per the distribution object + + pMesh pumi_mesh_loadSerial(pGeom g, const char* filename, const char* mesh_type) { if (strcmp(mesh_type,"mds")) { - if (!PCU_Comm_Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<getPCU()->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<getPCU()->GetMPIComm(); + int num_target_part = pumi::instance()->getPCU()->Peers(); + bool isMaster = ((pumi::instance()->getPCU()->Self() % num_target_part) == 0); pMesh m = 0; - split_comm(num_target_part); + split_comm(num_target_part, *pumi::instance()->getPCU()); if (isMaster) - m = apf::loadMdsMesh(g->getGmi(), filename); - merge_comm(prevComm); - pumi::instance()->mesh = expandMdsMesh(m, g->getGmi(), 1); + m = apf::loadMdsMesh(g->getGmi(), filename, pumi::instance()->getPCU()); + merge_comm(prevComm, *pumi::instance()->getPCU()); + pumi::instance()->mesh = expandMdsMesh(m, g->getGmi(), 1, m->getPCU()); return pumi::instance()->mesh; } @@ -228,50 +231,50 @@ pMesh pumi_mesh_load(pMesh m) return pumi::instance()->mesh; } - pMesh pumi_mesh_load(pGeom g, const char* filename, int num_in_part, const char* mesh_type) { if (strcmp(mesh_type,"mds")) { - if (!PCU_Comm_Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<getPCU()->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<1) // do static partitioning + if (num_in_part==1 && pumi::instance()->getPCU()->Peers()>1) // do static partitioning { - MPI_Comm prevComm = PCU_Get_Comm(); - int num_target_part = PCU_Comm_Peers()/num_in_part; - bool isMaster = ((PCU_Comm_Self() % num_target_part) == 0); + MPI_Comm prevComm = pumi::instance()->getPCU()->GetMPIComm(); + int num_target_part = pumi::instance()->getPCU()->Peers()/num_in_part; + bool isMaster = ((pumi::instance()->getPCU()->Self() % num_target_part) == 0); pMesh m = 0; apf::Migration* plan = 0; - split_comm(num_target_part); + split_comm(num_target_part, *pumi::instance()->getPCU()); if (isMaster) { - m = apf::loadMdsMesh(g->getGmi(), filename); + m = apf::loadMdsMesh(g->getGmi(), filename, pumi::instance()->getPCU()); plan = getPlan(m, num_target_part); } - merge_comm(prevComm); - pumi::instance()->mesh = apf::repeatMdsMesh(m, g->getGmi(), plan, num_target_part); + merge_comm(prevComm, *pumi::instance()->getPCU()); + pumi::instance()->mesh = apf::repeatMdsMesh(m, g->getGmi(), plan, num_target_part, pumi::instance()->getPCU()); } else - pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename); + pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename, pumi::instance()->getPCU()); pumi_mesh_print(pumi::instance()->mesh); return pumi::instance()->mesh; } + void send_entities(pMesh mesh, int dim) { - int local_id, self = PCU_Comm_Self(); + int local_id, self = mesh->getPCU()->Self(); pMeshEnt e; pMeshIter it = mesh->begin(dim); while ((e = mesh->iterate(it))) { local_id = getMdsIndex(mesh, e); - for (int pid=0; pidgetPCU()->Peers(); ++pid) { if (pid==self) continue; - PCU_Comm_Pack(pid, &local_id, sizeof(int)); - PCU_COMM_PACK(pid, e); + mesh->getPCU()->Pack(pid, &local_id, sizeof(int)); + mesh->getPCU()->Pack(pid, e); } } mesh->end(it); @@ -279,21 +282,22 @@ void send_entities(pMesh mesh, int dim) #include "apfMDS.h" #include "apfPM.h" + pMesh pumi_mesh_loadAll(pGeom g, const char* filename, bool stitch_link) { if (pumi_size()==1) - pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename); + pumi::instance()->mesh = apf::loadMdsMesh(g->getGmi(), filename, pumi::instance()->getPCU()); else { - double t0 = PCU_Time(); - MPI_Comm prevComm = PCU_Get_Comm(); - int num_target_part = PCU_Comm_Peers(); - split_comm(num_target_part); + double t0 = pcu::Time(); + MPI_Comm prevComm = pumi::instance()->getPCU()->GetMPIComm(); + int num_target_part = pumi::instance()->getPCU()->Peers(); + split_comm(num_target_part, *pumi::instance()->getPCU()); // no pmodel & remote links setup - pumi::instance()->mesh = apf::loadSerialMdsMesh(g->getGmi(), filename); - merge_comm(prevComm); - if (!PCU_Comm_Self()) - lion_oprint(1,"serial mesh %s loaded in %f seconds\n", filename, PCU_Time() - t0); + pumi::instance()->mesh = apf::loadSerialMdsMesh(g->getGmi(), filename, pumi::instance()->getPCU()); + merge_comm(prevComm, *pumi::instance()->getPCU()); + if (!pumi::instance()->getPCU()->Self()) + lion_oprint(1,"serial mesh %s loaded in %f seconds\n", filename, pcu::Time() - t0); } if (pumi_size()>1 && stitch_link) @@ -345,7 +349,7 @@ void pumi_mesh_setCount(pMesh m, pOwnership o) pumi::instance()->num_own_ent[dim] = n; } } - MPI_Allreduce(pumi::instance()->num_own_ent, pumi::instance()->num_global_ent, 4, MPI_INT, MPI_SUM, PCU_Get_Comm()); + MPI_Allreduce(pumi::instance()->num_own_ent, pumi::instance()->num_global_ent, 4, MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); #ifdef DEBUG if (!pumi_rank()) std::cout<<"[PUMI INFO] "<<__func__<<" end\n"; #endif @@ -357,7 +361,7 @@ int pumi_mesh_getNumEnt(pMesh m, int dim) int pumi_mesh_getNumOwnEnt(pMesh m, int dim) { PCU_ALWAYS_ASSERT(pumi::instance()->num_own_ent); - if (pumi::instance()->num_local_ent[dim]!=(int)m->count(dim) && !PCU_Comm_Self()) + if (pumi::instance()->num_local_ent[dim]!=(int)m->count(dim) && !m->getPCU()->Self()) { std::cout<<"[PUMI ERROR] "<<__func__<<": mesh count is not set. Please call pumi_mesh_setCount\n"; return -1; @@ -368,7 +372,7 @@ int pumi_mesh_getNumOwnEnt(pMesh m, int dim) int pumi_mesh_getNumGlobalEnt(pMesh m, int dim) { PCU_ALWAYS_ASSERT(pumi::instance()->num_global_ent); - if (pumi::instance()->num_local_ent[dim]!=(int)m->count(dim) && !PCU_Comm_Self()) + if (pumi::instance()->num_local_ent[dim]!=(int)m->count(dim) && !m->getPCU()->Self()) { std::cout<<"[PUMI ERROR] "<<__func__<<": mesh count is not set. Please call pumi_mesh_setCount\n"; return -1; @@ -407,12 +411,12 @@ void print_copies(pMesh m, pMeshEnt e) void pumi_mesh_print (pMesh m, bool print_ent) { - if (!PCU_Comm_Self()) std::cout<<"\n=== mesh size and tag info === \n"; + if (!m->getPCU()->Self()) std::cout<<"\n=== mesh size and tag info === \n"; - int* local_entity_count = new int[4*PCU_Comm_Peers()]; - int* own_entity_count = new int[4*PCU_Comm_Peers()]; + int* local_entity_count = new int[4*m->getPCU()->Peers()]; + int* own_entity_count = new int[4*m->getPCU()->Peers()]; - for (int i=0; i<4*PCU_Comm_Peers();++i) + for (int i=0; i<4*m->getPCU()->Peers();++i) local_entity_count[i]=own_entity_count[i]=0; pMeshEnt e; @@ -430,23 +434,23 @@ void pumi_mesh_print (pMesh m, bool print_ent) m->end(it); } - int* global_local_entity_count = new int[4*PCU_Comm_Peers()]; - int* global_own_entity_count = new int[4*PCU_Comm_Peers()]; + int* global_local_entity_count = new int[4*m->getPCU()->Peers()]; + int* global_own_entity_count = new int[4*m->getPCU()->Peers()]; - MPI_Allreduce(local_entity_count, global_local_entity_count, 4*PCU_Comm_Peers(), - MPI_INT, MPI_SUM, PCU_Get_Comm()); + MPI_Allreduce(local_entity_count, global_local_entity_count, 4*m->getPCU()->Peers(), + MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); - MPI_Allreduce(own_entity_count, global_own_entity_count, 4*PCU_Comm_Peers(), - MPI_INT, MPI_SUM, PCU_Get_Comm()); + MPI_Allreduce(own_entity_count, global_own_entity_count, 4*m->getPCU()->Peers(), + MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) { int* global_entity_count = new int[4]; global_entity_count[0]=global_entity_count[1]=global_entity_count[2]=global_entity_count[3]=0; for (int d=0; d<4;++d) { - for (int p=0; pgetPCU()->Peers();++p) global_entity_count[d] += global_own_entity_count[p*4+d]; } @@ -455,12 +459,12 @@ void pumi_mesh_print (pMesh m, bool print_ent) delete [] global_entity_count; - for (int p=0; pgetPCU()->Peers(); ++p) std::cout<<"(p"<getPCU()->Peers(); ++p) if (global_own_entity_count[p*4]) std::cout<<"(p"<getPCU()->Self()) { std::cout<<"mesh shape: \""<< m->getShape()->getName()<<"\"\n"; @@ -508,7 +512,7 @@ void pumi_mesh_print (pMesh m, bool print_ent) if (!m->findTag("global_id")) { pumi_mesh_createGlobalID(m); - if (!PCU_Comm_Self()) std::cout<<__func__<<": global id generated\n"; + if (!m->getPCU()->Self()) std::cout<<__func__<<": global id generated\n"; } int global_id; @@ -518,10 +522,10 @@ void pumi_mesh_print (pMesh m, bool print_ent) apf::Vector3 xyz; m->getPoint(e, 0, xyz); if (m->isGhost(e)) - std::cout<<"("<getPCU()->Self()<<") GHOST vtx "<getPCU()->Self()<<") vtx "<getDownward(e,d-1,down); if (m->isGhost(e)) - std::cout<<"("<getPCU()->Self()<<") GHOST e "<getPCU()->Self()<<") e "<getPCU()->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" failed: invalid mesh type "<getPCU()->Self()) { ++num_own_ent; if (own_copy!=e) @@ -753,7 +757,7 @@ void Distribution::print() ++i; if (parts_vec[i].size()==0) continue; APF_ITERATE(Parts,parts_vec[i],pit) - std::cout<<"("<getPCU()->Self()<<") distribute element "<end(it); @@ -780,7 +784,7 @@ static void distr_getAffected (pMesh m, Distribution* plan, EntityVector affecte for (int dimension=maxDimension-1; dimension >= 0; --dimension) { int upDimension = dimension + 1; - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(EntityVector,affected[upDimension],it) { pMeshEnt up = *it; @@ -796,21 +800,21 @@ static void distr_getAffected (pMesh m, Distribution* plan, EntityVector affecte Copies remotes; m->getRemotes(adjacent[i],remotes); APF_ITERATE(Copies,remotes,rit) - PCU_COMM_PACK(rit->first,rit->second); + m->getPCU()->Pack(rit->first,rit->second); if (m->hasMatching()) { apf::Matches matches; m->getMatches(adjacent[i],matches); for (size_t j=0; j < matches.getSize(); ++j) - PCU_COMM_PACK(matches[j].peer,matches[j].entity); + m->getPCU()->Pack(matches[j].peer,matches[j].entity); } }//downward adjacent loop }//upward affected loop - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) { pMeshEnt entity; - PCU_COMM_UNPACK(entity); + m->getPCU()->Unpack(entity); if ( !m->hasTag(entity,tag)) { m->setIntTag(entity,tag,&dummy); @@ -854,7 +858,7 @@ static void distr_updateResidences(pMesh m, for (int dimension = maxDimension-1; dimension >= 0; --dimension) { - PCU_Comm_Begin(); + m->getPCU()->Begin(); APF_ITERATE(EntityVector,affected[dimension],it) { pMeshEnt entity = *it; @@ -873,19 +877,19 @@ static void distr_updateResidences(pMesh m, m->getRemotes(entity,remotes); APF_ITERATE(Copies,remotes,rit) { - PCU_COMM_PACK(rit->first,rit->second); - apf::packParts(rit->first,newResidence); + m->getPCU()->Pack(rit->first,rit->second); + apf::packParts(rit->first,newResidence, m->getPCU()); } } - PCU_Comm_Send(); - while(PCU_Comm_Receive()) + m->getPCU()->Send(); + while(m->getPCU()->Receive()) { pMeshEnt entity; - PCU_COMM_UNPACK(entity); + m->getPCU()->Unpack(entity); Parts current; m->getResidence(entity,current); Parts incoming; - apf::unpackParts(incoming); + apf::unpackParts(incoming, m->getPCU()); apf::unite(current,incoming); m->setResidence(entity,current); } @@ -913,11 +917,11 @@ void distribute(pMesh m, Distribution* plan) void pumi_mesh_distribute(pMesh m, Distribution* plan) // ********************************************************* { - if (PCU_Comm_Peers()==1) return; + if (m->getPCU()->Peers()==1) return; if (pumi::instance()->ghosted_tag) { - if (!PCU_Comm_Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" not supported with ghosted mesh\n"; + if (!m->getPCU()->Self()) std::cout<<"[PUMI ERROR] "<<__func__<<" not supported with ghosted mesh\n"; return; } distribute(m, plan); diff --git a/pumi/pumi_numbering.cc b/pumi/pumi_numbering.cc index e8997231b..870e2f88e 100644 --- a/pumi/pumi_numbering.cc +++ b/pumi/pumi_numbering.cc @@ -14,7 +14,6 @@ #include "apfNumbering.h" #include "apfNumberingClass.h" #include -#include #include //************************************ @@ -62,7 +61,7 @@ pNumbering pumi_numbering_createGlobal(pMesh m, const char* name, pShape s, pOwn if (!s) s= m->getShape(); n = numberOwnedNodes(m, name, s, o); - apf::globalize(n); + apf::globalize(n, m->getPCU()); apf::synchronizeFieldData(n->getData(), o, false); //synchronize(n, o); return n; } @@ -99,7 +98,7 @@ pNumbering pumi_numbering_createProcGrp ( pMesh m, const char* name, int num_proc_grp, int dim, pOwnership o) { - assert(PCU_Comm_Peers()%num_proc_grp==0); + assert(m->getPCU()->Peers()%num_proc_grp==0); pNumbering n = m->findNumbering(name); if (n) @@ -109,8 +108,8 @@ pNumbering pumi_numbering_createProcGrp ( return n; } - int self = PCU_Comm_Self(); - int pgrp_size = PCU_Comm_Peers()/num_proc_grp; + int self = m->getPCU()->Self(); + int pgrp_size = m->getPCU()->Peers()/num_proc_grp; int local_pgrpid = self/pgrp_size; // divide int pgrp_rank = self % pgrp_size; @@ -135,10 +134,10 @@ pNumbering pumi_numbering_createProcGrp ( } int* in = new int; - int* out_arr = new int[PCU_Comm_Peers()]; // out[i] has local_numOwnedPartBdryEnt of process i on all processes + int* out_arr = new int[m->getPCU()->Peers()]; // out[i] has local_numOwnedPartBdryEnt of process i on all processes *in = owned_node_cnt; - MPI_Allgather(in, 1, MPI_INT, out_arr, 1, MPI_INT, PCU_Get_Comm()); + MPI_Allgather(in, 1, MPI_INT, out_arr, 1, MPI_INT, m->getPCU()->GetMPIComm()); it = m->begin(dim); int nbr = 0; @@ -194,10 +193,10 @@ void pumi_numbering_print(pNumbering n, int pid) pMeshEnt e; int nnodes; pShape s = n->getShape(); - if (pid==-1) pid = PCU_Comm_Peers(); + if (pid==-1) pid = m->getPCU()->Peers(); for (int rank=0; rankgetPCU()->Self()) { for(int dd = 0; dd < m->getDimension(); ++dd) { @@ -208,7 +207,7 @@ void pumi_numbering_print(pNumbering n, int pid) { nnodes = n->countNodesOn(e); for (int node=0; node < nnodes; ++node) - std::cout<<"("<getPCU()->Self()<<") ent "<isGhost(e)?1:0)<<")\n"; } // while m->end(it); diff --git a/pumi/pumi_sys.cc b/pumi/pumi_sys.cc index b41be4ff3..04f128bc9 100644 --- a/pumi/pumi_sys.cc +++ b/pumi/pumi_sys.cc @@ -7,7 +7,6 @@ BSD license as described in the LICENSE file in the top-level directory. *******************************************************************************/ -#include #include "pumi.h" #include #include @@ -17,37 +16,30 @@ // 0- SYSTEM-LEVEL FUNCTIONS //************************************ //************************************ - -void pumi_start() -{ - PCU_Comm_Init(); -} - -void pumi_finalize(bool) -{ - PCU_Comm_Free(); +void pumi_load_pcu(pcu::PCU *PCUObj){ + pumi::instance()->initializePCU(PCUObj); } int pumi_size() { - return PCU_Comm_Peers(); + return pumi::instance()->getPCU()->Peers(); } int pumi_rank() { - return PCU_Comm_Self(); + return pumi::instance()->getPCU()->Self(); } -void pumi_sync(void) +void pumi_sync() { - MPI_Barrier(PCU_Get_Comm()); + MPI_Barrier(pumi::instance()->getPCU()->GetMPIComm()); } #include #include void pumi_printSys() { - if (PCU_Comm_Self()) return; + if (pumi::instance()->getPCU()->Self()) return; struct utsname u; if (uname(&u) == 0) lion_oprint(1,"[%s] %s %s %s %s %s\n\n", @@ -64,12 +56,12 @@ double pumi_getTime() double pumi_getMem() { - return PCU_GetMem(); + return pcu::GetMem(); } void pumi_printTimeMem(const char* msg, double time, double memory) { - if (!PCU_Comm_Self()) + if (!pumi::instance()->getPCU()->Self()) { lion_oprint(1,"%-20s %6.3f sec %7.3f MB \n", msg, time, memory); fflush(stdout); diff --git a/python_wrappers/apf.i b/python_wrappers/apf.i index 68cc8372e..283b32005 100644 --- a/python_wrappers/apf.i +++ b/python_wrappers/apf.i @@ -2,7 +2,7 @@ %{ #include #include -#include +#include #include #include #include diff --git a/ree/ree.h b/ree/ree.h index 5e337484c..93c6147ff 100644 --- a/ree/ree.h +++ b/ree/ree.h @@ -15,7 +15,6 @@ #include "apf.h" #include -#include #include #include "apfShape.h" #include "apfField.h" diff --git a/ree/reeEstimateError.cc b/ree/reeEstimateError.cc index c3a4ab060..183713d0e 100644 --- a/ree/reeEstimateError.cc +++ b/ree/reeEstimateError.cc @@ -431,13 +431,13 @@ apf::Field* computeErrorField(apf::Field* ef, apf::Field* correctedFlux) apf::Field* estimateError(apf::Field* f) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); apf::Field* g = ree::equilibrateResiduals(f); lion_eprint(1,"1/4: residuals equilibrated \n"); apf::Field* theta = ree::computeFluxCorrection(f, g); lion_eprint(1,"2/4: flux corrections computed \n"); apf::destroyField(g); - PCU_Barrier(); + f->getMesh()->getPCU()->Barrier(); apf::Field* correctedFlux = ree::computeCorrectedFlux(f, theta); lion_eprint(1,"3/4: corrected flux field computed\n"); @@ -447,8 +447,8 @@ apf::Field* estimateError(apf::Field* f) lion_eprint(1,"4/4: error computed \n"); apf::destroyField(correctedFlux); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!f->getMesh()->getPCU()->Self()) lion_eprint(1,"REE: Error estimated in %f seconds\n",t1-t0); return error_field; diff --git a/ree/reeResidualFunctionals.cc b/ree/reeResidualFunctionals.cc index f26adea88..3016071b5 100644 --- a/ree/reeResidualFunctionals.cc +++ b/ree/reeResidualFunctionals.cc @@ -921,4 +921,4 @@ apf::Field* equilibrateResiduals(apf::Field* f) } -} +} \ No newline at end of file diff --git a/ree/reeSizeField.cc b/ree/reeSizeField.cc index 00765a01a..d2c483b15 100644 --- a/ree/reeSizeField.cc +++ b/ree/reeSizeField.cc @@ -156,14 +156,14 @@ apf::Field* getTargetEMSizeField( double alpha /*= 0.25*/, double beta /*= 2.0*/) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); Sizefield sz; setupSizefield(&sz, ef, error_field, n, alpha, beta); getElementSizeField(&sz); averageSizeField(&sz); apf::destroyField(sz.element_size); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!ef->getMesh()->getPCU()->Self()) lion_eprint(1,"EM: SizeField computed in %f seconds\n",t1-t0); return sz.size; } diff --git a/sam/samElementCount.cc b/sam/samElementCount.cc index 024199d9c..a5ac6086a 100644 --- a/sam/samElementCount.cc +++ b/sam/samElementCount.cc @@ -2,7 +2,6 @@ #include #include -#include namespace sam { @@ -52,8 +51,8 @@ class TotalMetricVolumeIso : public apf::Integrator { double vhat = getVolumeChange(dim, h); sum += vhat * w * dV; } - virtual void parallelReduce() { - sum = PCU_Add_Double(sum); + virtual void parallelReduce(pcu::PCU *PCUObj) { + sum = PCUObj->Add(sum); } }; diff --git a/spr/spr.h b/spr/spr.h index 19cae047c..d70978e3b 100644 --- a/spr/spr.h +++ b/spr/spr.h @@ -16,6 +16,8 @@ #include "apfNew.h" #include "apfDynamicVector.h" #include "apfDynamicMatrix.h" +#include "PCU.h" + /** \namespace spr * \brief All SPR error estimator functions @@ -43,7 +45,7 @@ apf::Field* recoverField(apf::Field* ip_field); * scales the output size field. * @returns a scalar mesh size field defined at vertices */ -apf::Field* getSPRSizeField(apf::Field* f, double adapt_ratio); +apf::Field* getSPRSizeField(apf::Field* f, double adapt_ratio, pcu::PCU *PCUObj); /** @brief run the SPR ZZ error estimator with a target # of output elems * @param f the integration-point input field @@ -55,6 +57,7 @@ apf::Field* getSPRSizeField(apf::Field* f, double adapt_ratio); apf::Field* getTargetSPRSizeField( apf::Field* f, size_t t, + pcu::PCU *PCUObj, double alpha=0.25, double beta=2.0); diff --git a/spr/sprEstimateError.cc b/spr/sprEstimateError.cc index 14aeaa592..523ef4134 100644 --- a/spr/sprEstimateError.cc +++ b/spr/sprEstimateError.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include #include "spr.h" @@ -25,9 +24,9 @@ class SInt : public apf::Integrator SInt(int order): apf::Integrator(order),r(0) {} - void parallelReduce() + void parallelReduce(pcu::PCU *PCUObj) { - PCU_Add_Doubles(&r,1); + PCUObj->Add(&r,1); } void reset() {r=0;} double r; @@ -309,14 +308,14 @@ static void estimateError(Estimation* e) apf::destroyField(e->element_size); } -apf::Field* getSPRSizeField(apf::Field* eps, double adaptRatio) +apf::Field* getSPRSizeField(apf::Field* eps, double adaptRatio, pcu::PCU *PCUObj) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); Estimation e; setupEstimation(&e, eps, adaptRatio); estimateError(&e); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!PCUObj->Self()) lion_eprint(1,"SPR: error estimated in %f seconds\n",t1-t0); return e.size; } diff --git a/spr/sprEstimateTargetError.cc b/spr/sprEstimateTargetError.cc index 0e05f3420..84b3183d1 100644 --- a/spr/sprEstimateTargetError.cc +++ b/spr/sprEstimateTargetError.cc @@ -11,8 +11,6 @@ and engineering 195.48 (2006): 6626-6645. */ #include "spr.h" - -#include #include #include #include @@ -66,9 +64,9 @@ class ScalarIntegrator : public apf::Integrator result(0) { } - void parallelReduce() + void parallelReduce(pcu::PCU *PCUObj) { - PCU_Add_Doubles(&result,1); + PCUObj->Add(&result,1); } double result; }; @@ -243,17 +241,18 @@ static void estimateError(Estimation* e) apf::Field* getTargetSPRSizeField( apf::Field* eps, size_t target, + pcu::PCU *PCUObj, double alpha, double beta) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); PCU_ALWAYS_ASSERT(target > 0); PCU_ALWAYS_ASSERT(alpha < beta); target::Estimation e; target::setupEstimation(&e, eps, target, alpha, beta); target::estimateError(&e); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!PCUObj->Self()) lion_eprint(1, "SPR (target): error estimated in %f seconds\n",t1-t0); return e.vtx_size; } diff --git a/spr/sprRecoverField.cc b/spr/sprRecoverField.cc index ef55aa84b..38490da1f 100644 --- a/spr/sprRecoverField.cc +++ b/spr/sprRecoverField.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include "spr.h" diff --git a/stk/apfExodusOutput.cc b/stk/apfExodusOutput.cc index 24e24b184..637d8ff0a 100644 --- a/stk/apfExodusOutput.cc +++ b/stk/apfExodusOutput.cc @@ -10,7 +10,6 @@ #error "configuration bug" #endif #include "apfSTK.h" -#include "PCU.h" #include #include #include @@ -53,14 +52,14 @@ void writeExodus( } if (bulk.is_null()) { - bulk = Teuchos::rcp(new stk::mesh::BulkData(*meta, PCU_Get_Comm())); + bulk = Teuchos::rcp(new stk::mesh::BulkData(*meta, mesh->getPCU()->GetMPIComm())); apf::copyMeshToBulk(n, models, meta.get(), bulk.get()); } apf::copyFieldsToBulk(n, meta.get(), bulk.get()); if (mesh_data.is_null()) { Ioss::Init::Initializer(); - mesh_data = Teuchos::rcp(new stk::io::StkMeshIoBroker(PCU_Get_Comm())); + mesh_data = Teuchos::rcp(new stk::io::StkMeshIoBroker(mesh->getPCU()->GetMPIComm())); output_file_idx = mesh_data->create_output_mesh(filename, stk::io::WRITE_RESULTS); mesh_data->set_bulk_data(*bulk); diff --git a/stk/apfSTK.cc b/stk/apfSTK.cc index d0dc5d853..b89479cc7 100644 --- a/stk/apfSTK.cc +++ b/stk/apfSTK.cc @@ -5,7 +5,6 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include #include #include #include "apfAlbany.h" @@ -769,7 +768,7 @@ long getStkId(GlobalNumbering* numbers, Node node) StkModels* create_sets(Mesh* m, const char* filename) { StkModels* sets = new StkModels; - if (! PCU_Comm_Self()) + if (! m->getPCU()->Self()) lion_oprint(1,"reading association file: %s\n", filename); static std::string const setNames[3] = { "node set", "side set", "elem set"}; diff --git a/test/1d.cc b/test/1d.cc index d43b828e7..654eab63d 100644 --- a/test/1d.cc +++ b/test/1d.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -8,11 +7,11 @@ #include #include -void createMesh(gmi_model*& g, apf::Mesh2*& m, int n) +void createMesh(gmi_model*& g, apf::Mesh2*& m, int n, pcu::PCU *PCUObj) { gmi_register_null(); g = gmi_load(".null"); - m = apf::makeEmptyMdsMesh(g, 1, false); + m = apf::makeEmptyMdsMesh(g, 1, false, PCUObj); apf::ModelEntity* left = m->findModelEntity(0, 0); apf::ModelEntity* right = m->findModelEntity(0, 1); apf::ModelEntity* mid = m->findModelEntity(1, 1); @@ -66,19 +65,20 @@ int main(int argc, char** argv) * 3 = mesh output name **/ PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_model* g; apf::Mesh2* m; int nverts = atoi(argv[1]); PCU_ALWAYS_ASSERT(2 <= nverts); PCU_ALWAYS_ASSERT(nverts <= 1000); - createMesh(g,m,nverts); + createMesh(g,m,nverts,&pcu_obj); test(m); gmi_write_dmg(g,argv[2]); m->writeNative(argv[3]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/H1Shapes.cc b/test/H1Shapes.cc index d69b978f1..ece447aee 100644 --- a/test/H1Shapes.cc +++ b/test/H1Shapes.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -29,12 +28,13 @@ void testH1( int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc != 3) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -44,12 +44,12 @@ int main(int argc, char** argv) gmi_register_null(); gmi_model* g = gmi_load(argv[1]); - apf::Mesh2* m = apf::loadMdsMesh(g,argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(g,argv[2],&PCUObj); m->verify(); // test fields interpolating a user-defined vector field for (int i = 1; i <= 6; i++) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.Self()) lion_oprint(1, "----TESTING VECTOR FIELD OF ORDER %d----\n", i); testH1( m, /* mesh */ @@ -61,7 +61,7 @@ int main(int argc, char** argv) // test fields interpolating a user-defined matrix field for (int i = 1; i <= 6; i++) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.Self()) lion_oprint(1, "----TESTING MATRIX FIELD OF ORDER %d----\n", i); testH1( m, /* mesh */ @@ -72,7 +72,7 @@ int main(int argc, char** argv) } apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -140,12 +140,12 @@ void testH1( for (int d = 0; d <= dim; d++) { if (!h1Field->getShape()->countNodesOn(apf::Mesh::simplexTypes[d])) { - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "no nodes in dimension %d\n", d); continue; } else - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "computing dofs for dimension %d\n", d); it = m->begin(d); while( (ent = m->iterate(it)) ) { @@ -210,7 +210,7 @@ void testH1( m->end(it); // check for field interpolation - if(0==PCU_Comm_Self()) { + if(0==m->getPCU()->Self()) { lion_oprint(1, "L2Error for entities of dimension %d is %e\n", d, L2Error); } PCU_ALWAYS_ASSERT_VERBOSE(L2Error < 1.e-12, diff --git a/test/L2Shapes.cc b/test/L2Shapes.cc index a2db1e071..9f7525769 100644 --- a/test/L2Shapes.cc +++ b/test/L2Shapes.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -32,12 +31,13 @@ void testL2writeNative( int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc != 3) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -47,7 +47,7 @@ int main(int argc, char** argv) gmi_register_null(); gmi_model* g = gmi_load(argv[1]); - apf::Mesh2* m = apf::loadMdsMesh(g,argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(g,argv[2],&PCUObj); m->verify(); for (int i = 0; i <= 6; i++) { @@ -61,7 +61,7 @@ int main(int argc, char** argv) testL2writeNative(m, 3, 3); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -96,12 +96,12 @@ void testL2( for (int d = 0; d <= dim; d++) { if (!l2Field->getShape()->countNodesOn(apf::Mesh::simplexTypes[d])) { - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "no nodes in dimension %d\n", d); continue; } else - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "computing dofs for dimension %d\n", d); it = m->begin(d); while( (ent = m->iterate(it)) ) { @@ -147,7 +147,7 @@ void testL2( m->end(it); // check for field interpolation - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "L2ErrorE is %e\n", L2ErrorE); PCU_ALWAYS_ASSERT_VERBOSE(L2ErrorE < 1.e-16, "Fields were not interpolated correctly!"); @@ -169,12 +169,12 @@ void testL2writeNative( for (int d = 0; d <= dim; d++) { if (!l2Field->getShape()->countNodesOn(apf::Mesh::simplexTypes[d])) { - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "no nodes in dimension %d\n", d); continue; } else - if(0==PCU_Comm_Self()) + if(0==m->getPCU()->Self()) lion_oprint(1, "computing dofs for dimension %d\n", d); it = m->begin(d); while( (ent = m->iterate(it)) ) { @@ -204,10 +204,10 @@ void testL2writeNative( // 2- read the mesh back in and make sure fields are on the mesh // 3- clean up the newly loaded mesh m->writeNative("L2Shape_test_mesh.smb"); - apf::Mesh2* m2 = apf::loadMdsMesh(".null", "./L2Shape_test_mesh.smb"); + apf::Mesh2* m2 = apf::loadMdsMesh(".null", "./L2Shape_test_mesh.smb", m->getPCU()); int fCount = 0; for (int i = 0; i < m2->countFields(); i++) { - if(0==PCU_Comm_Self()) + if(0==m2->getPCU()->Self()) lion_oprint(1, "field %d's name and shape are %s and %s\n", i, m2->getField(i)->getName(), m2->getField(i)->getShape()->getName()); fCount++; diff --git a/test/align.cc b/test/align.cc index 8cca65a43..83166344d 100644 --- a/test/align.cc +++ b/test/align.cc @@ -2,18 +2,17 @@ #include #include #include -#include #include #include #include -void testTriEdge() +void testTriEdge(pcu::PCU *PCUObj) { int which, rotate; bool flip; for(int ed = 0; ed < 6; ++ed){ gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); apf::MeshEntity* v[3]; for (int i = 0; i < 3; ++i) v[i] = m->createVert(0); @@ -32,13 +31,13 @@ void testTriEdge() apf::destroyMesh(m); } } -void testTetEdge() +void testTetEdge(pcu::PCU *PCUObj) { int which, rotate; bool flip; for(int ed = 0; ed < 12; ++ed){ gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::MeshEntity* v[4]; for (int i = 0; i < 4; ++i) v[i] = m->createVert(0); @@ -57,7 +56,7 @@ void testTetEdge() apf::destroyMesh(m); } } -void testTetTri() +void testTetTri(pcu::PCU *PCUObj) { int which, rotate; bool flip; @@ -65,7 +64,7 @@ void testTetTri() for(int flipped = 0; flipped < 2; ++flipped){ for(int fa = 0; fa < 12; ++fa){ gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::MeshEntity* v[4]; for (int i = 0; i < 4; ++i) v[i] = m->createVert(0); @@ -91,12 +90,13 @@ void testTetTri() int main() { MPI_Init(0,0); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_null(); - testTriEdge(); - testTetEdge(); - testTetTri(); - PCU_Comm_Free(); + testTriEdge(&pcu_obj); + testTetEdge(&pcu_obj); + testTetTri(&pcu_obj); + } MPI_Finalize(); } diff --git a/test/aniso_ma_test.cc b/test/aniso_ma_test.cc index 4eb9a0278..499dbfbf5 100644 --- a/test/aniso_ma_test.cc +++ b/test/aniso_ma_test.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -46,10 +45,11 @@ int main(int argc, char** argv) const char* meshFile = argv[2]; bool logInterpolation = atoi(argv[3]) > 0 ? true : false; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); m->verify(); apf::writeVtkFiles("aniso_before",m); AnIso sf(m); @@ -67,7 +67,7 @@ int main(int argc, char** argv) apf::writeVtkFiles("aniso_after",m); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ansys.cc b/test/ansys.cc index c7c41e64a..e9573b084 100644 --- a/test/ansys.cc +++ b/test/ansys.cc @@ -2,29 +2,29 @@ #include #include #include -#include #include #include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); - apf::Mesh2* m = apf::loadMdsFromANSYS(argv[1], argv[2]); + apf::Mesh2* m = apf::loadMdsFromANSYS(argv[1], argv[2], &pcu_obj); m->verify(); gmi_write_dmg(m->getModel(), argv[3]); m->writeNative(argv[4]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/assert_timing.cc b/test/assert_timing.cc index 4692dd830..699f0441d 100644 --- a/test/assert_timing.cc +++ b/test/assert_timing.cc @@ -7,22 +7,22 @@ #include double check_c_assert() { - double t0 = PCU_Time(); + double t0 = pcu::Time(); for (int i = 0; i < 100000000; ++i) { assert(pow((double)i, 1.01) < pow((double)i +1.0, 1.02)); } - double t1 = PCU_Time(); + double t1 = pcu::Time(); return t1-t0; } double check_pcu_assert() { - double t0 = PCU_Time(); + double t0 = pcu::Time(); for (int i = 0; i < 100000000; ++i) { double j = (double)i; double k = (double)i + 1.0; PCU_ALWAYS_ASSERT(pow(j, 1.01) < pow(k, 1.02)); } - double t1 = PCU_Time(); + double t1 = pcu::Time(); return t1-t0; } @@ -30,7 +30,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 2); int opt = atoi(argv[1]); MPI_Init(0,0); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); /* i'm avoiding conditionals inside for loops b/c i'm paranoid about the timings even though timings @@ -41,7 +42,7 @@ int main(int argc, char** argv) { else for (int i = 0; i < 5; ++i) printf("pcu assert in %f seconds\n", check_pcu_assert()); - PCU_Comm_Free(); + } MPI_Finalize(); return 0; } diff --git a/test/balance.cc b/test/balance.cc index 8b46af5a1..68efd60e4 100644 --- a/test/balance.cc +++ b/test/balance.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -11,11 +10,12 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); apf::MeshTag* weights = Parma_WeighByMemory(m); double step = 0.1; int verbose=1; apf::Balancer* balancer = Parma_MakeCentroidDiffuser(m, step, verbose); @@ -27,6 +27,6 @@ int main(int argc, char** argv) // destroy mds m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/bezierElevation.cc b/test/bezierElevation.cc index 8c5e1386b..2bb018624 100644 --- a/test/bezierElevation.cc +++ b/test/bezierElevation.cc @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -131,10 +130,10 @@ gmi_model* makeFaceModel() return model; } -apf::Mesh2* createMesh2D() +apf::Mesh2* createMesh2D(pcu::PCU *PCUObj) { gmi_model* model = makeFaceModel(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); apf::MeshEntity* v[3], *edges[3]; apf::Vector3 points2D[3] = {apf::Vector3(0,0,0),apf::Vector3(1,0,0),apf::Vector3(1,1,0)}; @@ -162,10 +161,10 @@ static apf::Vector3 points3D[4] = apf::Vector3(0,0,1)}; -apf::Mesh2* createMesh3D() +apf::Mesh2* createMesh3D(pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::buildOneElement(m,0,apf::Mesh::TET,points3D); apf::deriveMdsModel(m); @@ -175,12 +174,12 @@ apf::Mesh2* createMesh3D() return m; } -void testEdgeElevation() +void testEdgeElevation(pcu::PCU *PCUObj) { for (int o = 1; o <= 5; ++o){ gmi_model* model = makeEdgeModel(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 1, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 1, false, PCUObj); apf::Vector3 points[2] = {apf::Vector3(0,0,0),apf::Vector3(1,1,1)}; apf::MeshEntity* v[2]; @@ -233,12 +232,12 @@ void testEdgeElevation() } } -void testTriElevation() +void testTriElevation(pcu::PCU *PCUObj) { for (int o = 1; o <= 5; ++o){ - apf::Mesh2* m = createMesh2D(); + apf::Mesh2* m = createMesh2D(PCUObj); crv::BezierCurver bc(m,o,0); bc.run(); apf::MeshIterator* it = m->begin(2); @@ -304,13 +303,13 @@ void testTriElevation() } } -void testTetElevation() +void testTetElevation(pcu::PCU *PCUObj) { gmi_register_null(); for (int order = 1; order <= 4; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc(m,order,0); bc.run(); @@ -399,11 +398,12 @@ void testTetElevation() int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - testEdgeElevation(); - testTriElevation(); - testTetElevation(); - PCU_Comm_Free(); + testEdgeElevation(&pcu_obj); + testTriElevation(&pcu_obj); + testTetElevation(&pcu_obj); + } MPI_Finalize(); } diff --git a/test/bezierMesh.cc b/test/bezierMesh.cc index bc6995a77..fbf317c3f 100644 --- a/test/bezierMesh.cc +++ b/test/bezierMesh.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -121,10 +120,10 @@ gmi_model* makeModel() return model; } -apf::Mesh2* createMesh2D() +apf::Mesh2* createMesh2D(pcu::PCU *PCUObj) { gmi_model* model = makeModel(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); apf::MeshEntity* v[4]; apf::Vector3 points2D[4] = {apf::Vector3(0,0,0), @@ -153,10 +152,10 @@ apf::Mesh2* createMesh2D() return m; } -apf::Mesh2* createMesh3D() +apf::Mesh2* createMesh3D(pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::Vector3 points3D[4] = {apf::Vector3(0,0,0), @@ -253,12 +252,12 @@ void testSize2D(apf::Mesh2* m, int order) } } -void test2D() +void test2D(pcu::PCU *PCUObj) { // test all orders for all blending orders for(int order = 1; order <= 6; ++order){ for(int blendOrder = 0; blendOrder <= 2; ++blendOrder){ - apf::Mesh2* m = createMesh2D(); + apf::Mesh2* m = createMesh2D(PCUObj); crv::BezierCurver bc(m,order,blendOrder); bc.run(); testInterpolatedPoints2D(m); @@ -269,7 +268,7 @@ void test2D() } // test the pure interpolation side of things { - apf::Mesh2* m = createMesh2D(); + apf::Mesh2* m = createMesh2D(PCUObj); apf::changeMeshShape(m,apf::getLagrange(2),true); crv::InterpolatingCurver ic(m,2); ic.run(); @@ -467,10 +466,10 @@ void test3D(apf::Mesh2* m) * triangles implemented. There are no nodes inside the tetrahedron * */ -void test3DBlended() +void test3DBlended(pcu::PCU *PCUObj) { gmi_register_null(); - apf::Mesh2* mbase = createMesh3D(); + apf::Mesh2* mbase = createMesh3D(PCUObj); testSize3D(mbase); test3DJacobian(mbase); @@ -481,7 +480,7 @@ void test3DBlended() for(int order = 1; order <= 6; ++order){ for(int blendOrder = 1; blendOrder <= 2; ++blendOrder){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc(m,order,blendOrder); bc.run(); test3D(m); @@ -492,12 +491,12 @@ void test3DBlended() } /* Tests Full Bezier tetrahedra */ -void test3DFull() +void test3DFull(pcu::PCU *PCUObj) { gmi_register_null(); for(int order = 1; order <= 6; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc(m,order,0); bc.run(); test3D(m); @@ -540,7 +539,7 @@ void test3DFull() } // test simple elevation for(int order = 1; order <= 4; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc(m,order,0); bc.run(); crv::changeMeshOrder(m,5); @@ -550,7 +549,7 @@ void test3DFull() } // test elevation inside a BezierCurver for(int order = 1; order <= 4; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc1(m,order,0); bc1.run(); crv::BezierCurver bc2(m,order+2,0); @@ -561,7 +560,7 @@ void test3DFull() } // test going downward for(int order = 4; order <= 6; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc1(m,order,0); bc1.run(); crv::BezierCurver bc2(m,order-2,0); @@ -572,7 +571,7 @@ void test3DFull() } // test going from 2nd order lagrange to various order bezier for(int order = 2; order <= 6; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); apf::changeMeshShape(m,apf::getLagrange(2),true); crv::BezierCurver bc(m,order,0); bc.run(); @@ -585,11 +584,12 @@ void test3DFull() int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - test2D(); - test3DBlended(); - test3DFull(); - PCU_Comm_Free(); + test2D(&pcu_obj); + test3DBlended(&pcu_obj); + test3DFull(&pcu_obj); + } MPI_Finalize(); } diff --git a/test/bezierMisc.cc b/test/bezierMisc.cc index 77c4b0d95..2f3b389c2 100644 --- a/test/bezierMisc.cc +++ b/test/bezierMisc.cc @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -25,10 +24,10 @@ static apf::Vector3 points3D[4] = apf::Vector3(0,1,0), apf::Vector3(0,0,1)}; -apf::Mesh2* createMesh2D() +apf::Mesh2* createMesh2D(pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); apf::buildOneElement(m,0,apf::Mesh::TRIANGLE,points3D); apf::deriveMdsModel(m); @@ -38,10 +37,10 @@ apf::Mesh2* createMesh2D() return m; } -apf::Mesh2* createMesh3D() +apf::Mesh2* createMesh3D(pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::buildOneElement(m,0,apf::Mesh::TET,points3D); apf::deriveMdsModel(m); @@ -167,11 +166,12 @@ void testMatrixInverse(){ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); testNodeIndexing(); testMatrixInverse(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/bezierRefine.cc b/test/bezierRefine.cc index 681930071..f413108b2 100644 --- a/test/bezierRefine.cc +++ b/test/bezierRefine.cc @@ -9,7 +9,6 @@ #include #include #include -#include #include #include @@ -138,10 +137,10 @@ gmi_model* makeModel() return model; } -apf::Mesh2* createMesh2D() +apf::Mesh2* createMesh2D(pcu::PCU *PCUObj) { gmi_model* model = makeModel(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); apf::MeshEntity* v[4]; apf::Vector3 points2D[4] = {apf::Vector3(0,0,0), @@ -183,10 +182,10 @@ static double measureMesh(apf::Mesh2* m) return v; } -void test2D() +void test2D(pcu::PCU *PCUObj) { for(int order = 1; order <= 6; ++order){ - apf::Mesh2* m = createMesh2D(); + apf::Mesh2* m = createMesh2D(PCUObj); apf::changeMeshShape(m, crv::getBezier(order),true); crv::BezierCurver bc(m,order,0); if(order > 1){ @@ -239,10 +238,10 @@ void test2D() } } -apf::Mesh2* createMesh3D() +apf::Mesh2* createMesh3D(pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); double x = 1./sqrt(6.); double z = 1./sqrt(12.); apf::Vector3 points3D[4] = @@ -260,12 +259,12 @@ apf::Mesh2* createMesh3D() return m; } -void test3D() +void test3D(pcu::PCU *PCUObj) { gmi_register_null(); // test full for(int order = 1; order <= 4; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc(m,order,0); bc.run(); @@ -285,7 +284,7 @@ void test3D() } // test blended for(int order = 1; order <= 4; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc(m,order,1); bc.run(); @@ -311,10 +310,11 @@ void test3D() int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - test2D(); - test3D(); - PCU_Comm_Free(); + test2D(&pcu_obj); + test3D(&pcu_obj); + } MPI_Finalize(); } diff --git a/test/bezierShapeEval.cc b/test/bezierShapeEval.cc index 07abf4785..ba6b5136d 100644 --- a/test/bezierShapeEval.cc +++ b/test/bezierShapeEval.cc @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -20,17 +19,18 @@ * blended bezier shapes */ -static apf::Mesh2* makeOneTriMesh(int order, apf::MeshEntity* &ent); -static apf::Mesh2* makeOneTetMesh(int order, apf::MeshEntity* &ent); +static apf::Mesh2* makeOneTriMesh(int order, apf::MeshEntity* &ent, pcu::PCU *PCUObj); +static apf::Mesh2* makeOneTetMesh(int order, apf::MeshEntity* &ent, pcu::PCU *PCUObj); int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 7 ) { - if ( !PCU_Comm_Self() ) { + if ( !PCUObj.Self() ) { printf("Usage: %s >\n", argv[0]); printf(" can only be 2 (for triangles) and 4 (for tets)\n"); printf(" is the order of bezier\n"); @@ -50,7 +50,7 @@ int main(int argc, char** argv) " must be between -1 and 2!"); apf::MeshEntity* ent = 0; - apf::Mesh2* m = type == 2 ? makeOneTriMesh(order,ent) : makeOneTetMesh(order,ent); + apf::Mesh2* m = type == 2 ? makeOneTriMesh(order,ent,&PCUObj) : makeOneTetMesh(order,ent,&PCUObj); double xi0 = atof(argv[4]); double xi1 = atof(argv[5]); @@ -80,13 +80,13 @@ int main(int argc, char** argv) printf("%d, %E \n", i, vals[i]); } - PCU_Comm_Free(); + } MPI_Finalize(); } -static apf::Mesh2* makeOneTriMesh(int order, apf::MeshEntity* &ent) +static apf::Mesh2* makeOneTriMesh(int order, apf::MeshEntity* &ent, pcu::PCU *PCUObj) { - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(0, 2, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(0, 2, false, PCUObj); double vert_coords[3][6] = { {0.,0.,0., 0., 0., 0.}, @@ -142,10 +142,10 @@ static apf::Mesh2* makeOneTriMesh(int order, apf::MeshEntity* &ent) return mesh; } -static apf::Mesh2* makeOneTetMesh(int order, apf::MeshEntity* &ent) +static apf::Mesh2* makeOneTetMesh(int order, apf::MeshEntity* &ent, pcu::PCU *PCUObj) { - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(0, 3, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(0, 3, false, PCUObj); double vert_coords[4][6] = { {0.,0.,0., 0., 0., 0.}, diff --git a/test/bezierSubdivision.cc b/test/bezierSubdivision.cc index 9eb99d329..308722fe5 100644 --- a/test/bezierSubdivision.cc +++ b/test/bezierSubdivision.cc @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -131,10 +130,10 @@ gmi_model* makeFaceModel() return model; } -apf::Mesh2* createMesh2D() +apf::Mesh2* createMesh2D(pcu::PCU *PCUObj) { gmi_model* model = makeFaceModel(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); apf::MeshEntity* v[3], *edges[3]; apf::Vector3 points2D[3] = {apf::Vector3(0,0,0),apf::Vector3(1,0,0),apf::Vector3(1,1,0)}; @@ -162,10 +161,10 @@ static apf::Vector3 points3D[4] = apf::Vector3(0,0,1)}; -apf::Mesh2* createMesh3D() +apf::Mesh2* createMesh3D(pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::buildOneElement(m,0,apf::Mesh::TET,points3D); apf::deriveMdsModel(m); @@ -182,12 +181,12 @@ apf::Mesh2* createMesh3D() * to compare correctness of the split. * */ -void testEdgeSubdivision() +void testEdgeSubdivision(pcu::PCU *PCUObj) { for (int o = 1; o <= 6; ++o){ gmi_model* model = makeEdgeModel(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 1, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 1, false, PCUObj); apf::ModelEntity* edgeModel = m->findModelEntity(1,0); @@ -259,11 +258,11 @@ void testEdgeSubdivision() * exactly replicates its part of the old one * */ -void testTriSubdivision1() +void testTriSubdivision1(pcu::PCU *PCUObj) { for(int o = 1; o <= 6; ++o){ - apf::Mesh2* m = createMesh2D(); + apf::Mesh2* m = createMesh2D(PCUObj); crv::BezierCurver bc(m,o,0); bc.run(); @@ -347,10 +346,10 @@ void testTriSubdivision1() /* Create a single triangle, split it into 4, and try not to crash * */ -void testTriSubdivision4() +void testTriSubdivision4(pcu::PCU *PCUObj) { for(int o = 2; o <= 2; ++o){ - apf::Mesh2* m = createMesh2D(); + apf::Mesh2* m = createMesh2D(PCUObj); crv::BezierCurver bc(m,o,0); bc.run(); apf::MeshIterator* it = m->begin(2); @@ -379,14 +378,14 @@ void testTriSubdivision4() * exactly replicates its part of the old one * */ -void testTetSubdivision1() +void testTetSubdivision1(pcu::PCU *PCUObj) { gmi_register_null(); for (int order = 1; order <= 4; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); crv::BezierCurver bc(m,order,0); bc.run(); @@ -482,12 +481,13 @@ void testTetSubdivision1() int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - testEdgeSubdivision(); - testTriSubdivision1(); - testTriSubdivision4(); - testTetSubdivision1(); - PCU_Comm_Free(); + testEdgeSubdivision(&pcu_obj); + testTriSubdivision1(&pcu_obj); + testTriSubdivision4(&pcu_obj); + testTetSubdivision1(&pcu_obj); + } MPI_Finalize(); } diff --git a/test/bezierValidity.cc b/test/bezierValidity.cc index 694292114..9f627c3e0 100644 --- a/test/bezierValidity.cc +++ b/test/bezierValidity.cc @@ -9,7 +9,6 @@ #include #include #include -#include #include #include @@ -186,10 +185,10 @@ gmi_model* makeModel() return model; } -apf::Mesh2* createMesh2D() +apf::Mesh2* createMesh2D(pcu::PCU *PCUObj) { gmi_model* model = makeModel(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, PCUObj); apf::MeshEntity* v[4]; apf::Vector3 points2D[4] = {apf::Vector3(0,0,0), @@ -245,10 +244,10 @@ void checkValidity(apf::Mesh* m, int order) m->end(it); } -void test2D() +void test2D(pcu::PCU *PCUObj) { for(int order = 2; order <= 6; ++order){ - apf::Mesh2* m = createMesh2D(); + apf::Mesh2* m = createMesh2D(PCUObj); apf::changeMeshShape(m, crv::getBezier(order),true); crv::BezierCurver bc(m,order,0); // creates interpolation points based on the edges of the geometry @@ -299,10 +298,10 @@ void test2D() } } -apf::Mesh2* createMesh3D() +apf::Mesh2* createMesh3D(pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::Vector3 points3D[4] = {apf::Vector3(0,0,0), @@ -319,12 +318,12 @@ apf::Mesh2* createMesh3D() return m; } -void test3D() +void test3D(pcu::PCU *PCUObj) { gmi_register_null(); for(int order = 2; order <= 4; ++order){ - apf::Mesh2* m = createMesh3D(); + apf::Mesh2* m = createMesh3D(PCUObj); apf::changeMeshShape(m, crv::getBezier(order),true); apf::FieldShape* fs = m->getShape(); crv::BezierCurver bc(m,order,0); @@ -407,10 +406,11 @@ void test3D() int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - test2D(); - test3D(); - PCU_Comm_Free(); + test2D(&pcu_obj); + test3D(&pcu_obj); + } MPI_Finalize(); } diff --git a/test/box.cc b/test/box.cc index e317232d4..a5ed9b0dc 100644 --- a/test/box.cc +++ b/test/box.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -20,11 +19,11 @@ bool is = true; const char* modelFile = 0; const char* meshFile = 0; -void verifyArgs(int argc, char** argv) +void verifyArgs(int argc, char** argv, pcu::PCU *PCUObj) { if (argc != 10) { - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) { printf("Usage: %s " " \n", argv[0]); @@ -61,23 +60,24 @@ void getArgs(char** argv) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - verifyArgs(argc, argv); - if(PCU_Comm_Peers()>1) { - if(PCU_Comm_Self()) + verifyArgs(argc, argv, &pcu_obj); + if(pcu_obj.Peers()>1) { + if(pcu_obj.Self()) fprintf(stderr, "%s must be run on a single process!\n", argv[0]); exit(1); } getArgs(argv); gmi_register_mesh(); - apf::Mesh2* m = apf::makeMdsBox(nx,ny,nz,wx,wy,wz,is); + apf::Mesh2* m = apf::makeMdsBox(nx,ny,nz,wx,wy,wz,is,&pcu_obj); gmi_model* g = m->getModel(); m->verify(); m->writeNative(meshFile); gmi_write_dmg(g, modelFile); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/capVol.cc b/test/capVol.cc index a9149c585..ea2fa676f 100644 --- a/test/capVol.cc +++ b/test/capVol.cc @@ -30,7 +30,6 @@ namespace { void myExit(int exit_code = EXIT_SUCCESS) { gmi_cap_stop(); - PCU_Comm_Free(); MPI_Finalize(); exit(exit_code); } @@ -77,7 +76,8 @@ void printUsage(char *argv0) { int main(int argc, char** argv) { // Initialize parallelism. MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); // Initialize logging. lion_set_stdout(stdout); @@ -85,7 +85,7 @@ int main(int argc, char** argv) { // Check arguments or print usage. if (argc < 3) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.Self() == 0) { printUsage(argv[0]); } myExit(EXIT_FAILURE); @@ -173,7 +173,7 @@ int main(int argc, char** argv) { MG_API_CALL(m, compute_adjacency()); // Make APF adapter over Capstone mesh. - ma::Mesh* apfCapMesh = apf::createMesh(m, g); + ma::Mesh* apfCapMesh = apf::createMesh(m, g, &PCUObj); // Choose appropriate size-field. ma::AnisotropicFunction* sf = nullptr; @@ -271,7 +271,6 @@ int main(int argc, char** argv) { // Exit calls. gmi_cap_stop(); - PCU_Comm_Free(); + } MPI_Finalize(); -} - +} \ No newline at end of file diff --git a/test/cgns.cc b/test/cgns.cc index 6f853d2e0..d84dc96c5 100644 --- a/test/cgns.cc +++ b/test/cgns.cc @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include // @@ -24,7 +24,7 @@ apf::MeshTag *create_int_tag(const std::string &name, apf::Mesh2 *m, int dim) apf::MeshEntity *elem = nullptr; apf::MeshIterator *it = m->begin(dim); int vals[1]; - vals[0] = PCU_Comm_Self(); + vals[0] = m->getPCU()->Self(); while ((elem = m->iterate(it))) m->setIntTag(elem, tag, vals); m->end(it); @@ -68,7 +68,7 @@ void balance(const std::string &prefix, const apf::ZoltanMethod &method, apf::Me Parma_PrintPtnStats(m, ""); m->destroyTag(weights); - const std::string name = prefix + "_balance_" + std::to_string(PCU_Comm_Peers()) + "procs"; + const std::string name = prefix + "_balance_" + std::to_string(m->getPCU()->Peers()) + "procs"; apf::writeVtkFiles(name.c_str(), m); } @@ -103,7 +103,7 @@ void simpleReorder(const std::string &prefix, apf::Mesh2 *m) apf::makeGlobal(apf::numberElements(m, "elem Idx_pstOrder")); } - const std::string name = prefix + "_reorder_" + std::to_string(PCU_Comm_Peers()) + "procs"; + const std::string name = prefix + "_reorder_" + std::to_string(m->getPCU()->Peers()) + "procs"; apf::writeVtkFiles(name.c_str(), m); } @@ -114,7 +114,7 @@ pMesh toPumi(const std::string &prefix, gmi_model *g, apf::Mesh2 *mesh) pMesh pm = pumi_mesh_load(mesh); std::cout << pm << std::endl; pumi_mesh_verify(pm); - const std::string name = prefix + "_toPUMI_" + std::to_string(PCU_Comm_Peers()) + "procs"; + const std::string name = prefix + "_toPUMI_" + std::to_string(pm->getPCU()->Peers()) + "procs"; pumi_mesh_write(pm, name.c_str(), "vtk"); return pm; } @@ -154,7 +154,7 @@ auto additional(const std::string &prefix, gmi_model *g, apf::Mesh2 *mesh) } pm->end(it); - const std::string name = prefix + "_toPUMI_" + std::to_string(PCU_Comm_Peers()) + "procs"; + const std::string name = prefix + "_toPUMI_" + std::to_string(mesh->getPCU()->Peers()) + "procs"; // only the owned elements will have a elmField value of 1 pumi_mesh_write(pm, (name + "_beforeSync").c_str(), "vtk"); @@ -172,12 +172,14 @@ auto additional(const std::string &prefix, gmi_model *g, apf::Mesh2 *mesh) return clean; } -std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std::string &argv2, const std::string &post, const bool &additionalTests, const bool &writeCGNS, const std::vector> &meshData) +std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std::string &argv2, const std::string &post, const bool &additionalTests, const bool &writeCGNS, const std::vector> &meshData, pcu::PCU *PCUObj) { gmi_register_null(); gmi_register_mesh(); gmi_model *g = gmi_load(".null"); - apf::Mesh2 *m = apf::loadMdsFromCGNS(g, argv1.c_str(), cgnsBCMap, meshData); + PCU_t h; + h.ptr = static_cast(PCUObj); + apf::Mesh2 *m = apf::loadMdsFromCGNS(h, g, argv1.c_str(), cgnsBCMap, meshData); m->verify(); // m->writeNative(argv2.c_str()); @@ -200,45 +202,45 @@ std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std: if (dim == 3) { { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_cellMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_cellMesh"); apf::writeVtkFiles(name.c_str(), m, 3); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_faceMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_faceMesh"); apf::writeVtkFiles(name.c_str(), m, 2); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_edgeMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_edgeMesh"); apf::writeVtkFiles(name.c_str(), m, 1); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_vertexMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_vertexMesh"); apf::writeVtkFiles(name.c_str(), m, 0); } } else if (dim == 2) { { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_cellMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_cellMesh"); apf::writeVtkFiles(name.c_str(), m, 2); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_edgeMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_edgeMesh"); apf::writeVtkFiles(name.c_str(), m, 1); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_vertexMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_vertexMesh"); apf::writeVtkFiles(name.c_str(), m, 0); } } else if (dim == 1) { { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_cellMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_cellMesh"); apf::writeVtkFiles(name.c_str(), m, 1); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_vertexMesh"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_vertexMesh"); apf::writeVtkFiles(name.c_str(), m, 0); } } @@ -246,7 +248,7 @@ std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std: std::string cgnsOutputName; if (writeCGNS) { - cgnsOutputName = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + "_outputFile.cgns"; + cgnsOutputName = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + "_outputFile.cgns"; apf::writeCGNS(cgnsOutputName.c_str(), m, cgnsBCMap); } @@ -386,45 +388,45 @@ std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std: if (dim == 3) { { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_cellMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_cellMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 3); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_faceMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_faceMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 2); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_edgeMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_edgeMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 1); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_vertexMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_vertexMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 0); } } else if (dim == 2) { { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_cellMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_cellMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 2); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_edgeMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_edgeMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 1); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_vertexMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_vertexMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 0); } } else if (dim == 1) { { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_cellMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_cellMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 1); } { - const auto name = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + std::string("_toVTK_vertexMesh_additional"); + const auto name = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + std::string("_toVTK_vertexMesh_additional"); apf::writeVtkFiles(name.c_str(), m, 0); } } @@ -432,7 +434,7 @@ std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std: if (writeCGNS) { // what this one to be re-read if doing re-reading so that the dummy variables (vector/matrix/scalar) are there - cgnsOutputName = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + "_additional_outputFile.cgns"; + cgnsOutputName = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + "_additional_outputFile.cgns"; apf::writeCGNS(cgnsOutputName.c_str(), m, cgnsBCMap); } // @@ -454,7 +456,7 @@ std::string doit(apf::CGNSBCMap &cgnsBCMap, const std::string &argv1, const std: apf::displaceMesh(m, field); if (writeCGNS) { - std::string cgnsOutputName = prefix + "_" + std::to_string(PCU_Comm_Peers()) + "procs" + "_additional_moved_outputFile.cgns"; + std::string cgnsOutputName = prefix + "_" + std::to_string(m->getPCU()->Peers()) + "procs" + "_additional_moved_outputFile.cgns"; apf::writeCGNS(cgnsOutputName.c_str(), m, cgnsBCMap); } } @@ -477,12 +479,14 @@ int main(int argc, char **argv) { #ifdef HAVE_CGNS MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pumi_load_pcu(&PCUObj); lion_set_verbosity(1); bool additionalTests = false; if (argc < 3) { - if (!PCU_Comm_Self()) + if (!PCUObj.Self()) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -494,7 +498,7 @@ int main(int argc, char **argv) additionalTests = true; else { - if (!PCU_Comm_Self()) + if (!PCUObj.Self()) printf("Usage: %s additional\n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -503,7 +507,7 @@ int main(int argc, char **argv) } else if (argc > 4) { - if (!PCU_Comm_Self()) + if (!PCUObj.Self()) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -515,7 +519,7 @@ int main(int argc, char **argv) { apf::CGNSBCMap cgnsBCMap; std::vector> meshData; - cgnsOutputName = doit(cgnsBCMap, argv[1], argv[2], "", additionalTests, additionalTests, meshData); + cgnsOutputName = doit(cgnsBCMap, argv[1], argv[2], "", additionalTests, additionalTests, meshData, &PCUObj); } // Phase 2 if (additionalTests) @@ -538,10 +542,10 @@ int main(int argc, char **argv) meshData.push_back(std::make_pair("CellCenter", "DummyMatrix_1")); apf::CGNSBCMap cgnsBCMap; std::cout << "RE-READING " << std::endl; - doit(cgnsBCMap, cgnsOutputName.c_str(), "tempy.smb", "_reread", false, true, meshData); + doit(cgnsBCMap, cgnsOutputName.c_str(), "tempy.smb", "_reread", false, true, meshData, &PCUObj); } // - PCU_Comm_Free(); + } MPI_Finalize(); return 0; #else diff --git a/test/classifyThenAdapt.cc b/test/classifyThenAdapt.cc index a2d01f3d0..081ddf5c7 100644 --- a/test/classifyThenAdapt.cc +++ b/test/classifyThenAdapt.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -45,9 +44,9 @@ void createVtx(apf::Mesh2* m, int mdlDim, int mdlId, apf::Vector3 pt, EntVec& ve verts.push_back(vert); } -apf::Mesh2* createTriMesh() { +apf::Mesh2* createTriMesh(pcu::PCU *PCUObj) { gmi_model* g = gmi_load(".null"); - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(g, 2, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(g, 2, false, PCUObj); apf::Vector3 p0( 0.0, 1.0, 0.0); apf::Vector3 p1( 0.5, 1.0, 0.0); @@ -127,11 +126,12 @@ void printClassCounts(apf::Mesh* m) { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_null(); - ma::Mesh* m = createTriMesh(); + ma::Mesh* m = createTriMesh(&pcu_obj); m->verify(); // make sure the mesh is valid! gmi_model* g = m->getModel(); gmi_write_dmg(g,"model.dmg"); @@ -148,6 +148,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/collapse.cc b/test/collapse.cc index da45c1fd4..2ddced964 100644 --- a/test/collapse.cc +++ b/test/collapse.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -26,10 +25,10 @@ namespace { } }; - void getConfig(int argc, char** argv) + void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: mpirun -np %s \n" "Reduce the part count of mesh from inPartCount to inPartCount/factor.\n", argv[0]); @@ -40,25 +39,26 @@ namespace { meshFile = argv[2]; outFile = argv[3]; partitionFactor = atoi(argv[4]); - PCU_ALWAYS_ASSERT(partitionFactor <= PCU_Comm_Peers()); + PCU_ALWAYS_ASSERT(partitionFactor <= PCUObj->Peers()); } } int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - PCU_Protect(); + pcu::Protect(); #ifdef HAVE_SIMMETRIX Sim_readLicenseFile(0); gmi_sim_start(); gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv); + getConfig(argc,argv,&pcu_obj); GroupCode code; - code.mesh = apf::loadMdsMesh(modelFile, meshFile); - apf::Unmodulo outMap(PCU_Comm_Self(), PCU_Comm_Peers()); + code.mesh = apf::loadMdsMesh(modelFile, meshFile, &pcu_obj); + apf::Unmodulo outMap(pcu_obj.Self(), pcu_obj.Peers()); Parma_ShrinkPartition(code.mesh, partitionFactor, code); code.mesh->destroyNative(); apf::destroyMesh(code.mesh); @@ -66,6 +66,6 @@ int main(int argc, char** argv) { gmi_sim_stop(); Sim_unregisterAllKeys(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/construct.cc b/test/construct.cc index a702b973d..cfb5597bd 100644 --- a/test/construct.cc +++ b/test/construct.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include @@ -12,7 +11,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -22,7 +22,7 @@ int main(int argc, char** argv) int etype; int nverts; - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); int dim = m->getDimension(); extractCoords(m, coords, nverts); destruct(m, conn, nelem, etype); @@ -30,7 +30,7 @@ int main(int argc, char** argv) apf::destroyMesh(m); gmi_model* model = gmi_load(".null"); - m = apf::makeEmptyMdsMesh(model, dim, false); + m = apf::makeEmptyMdsMesh(model, dim, false, &PCUObj); apf::GlobalToVert outMap; apf::construct(m, conn, nelem, etype, outMap); delete [] conn; @@ -45,7 +45,7 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/constructThenGhost.cc b/test/constructThenGhost.cc index e90e41b34..a4d17cf1a 100644 --- a/test/constructThenGhost.cc +++ b/test/constructThenGhost.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -14,7 +13,9 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); + pumi_load_pcu(&pcu_obj); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -24,7 +25,7 @@ int main(int argc, char** argv) int etype; int nverts; - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2], &pcu_obj); int dim = m->getDimension(); extractCoords(m, coords, nverts); destruct(m, conn, nelem, etype); @@ -32,7 +33,7 @@ int main(int argc, char** argv) apf::destroyMesh(m); gmi_model* model = gmi_load(".null"); - m = apf::makeEmptyMdsMesh(model, dim, false); + m = apf::makeEmptyMdsMesh(model, dim, false, &pcu_obj); apf::GlobalToVert outMap; apf::construct(m, conn, nelem, etype, outMap); delete [] conn; @@ -46,7 +47,7 @@ int main(int argc, char** argv) if (!pumi_rank()) printf("model/mesh converted to pumi instance\n"); //create the pumi instance to use pumi api's - pGeom g=pumi_geom_load(model); + pGeom g = pumi_geom_load(model); pMesh pm = pumi_mesh_load(m); pumi_mesh_verify(pm); @@ -91,6 +92,6 @@ int main(int argc, char** argv) pumi_geom_delete(g); pumi_mesh_delete(pm); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/construct_bottom_up.cc b/test/construct_bottom_up.cc index 2856ee2c2..f982dff88 100644 --- a/test/construct_bottom_up.cc +++ b/test/construct_bottom_up.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include @@ -139,10 +138,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT_VERBOSE(PCU_Comm_Peers() == 1, "Not implemented in parallel!"); + PCU_ALWAYS_ASSERT_VERBOSE(PCUObj.Peers() == 1, "Not implemented in parallel!"); if (argc < 2) { printf("USAGE 1 ( no model): %s \n", argv[0]); printf("USAGE 2 (with model): %s destroyNative(); apf::destroyMesh(outMesh); - - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/convert.cc b/test/convert.cc index 6813e45b1..12e965c5a 100644 --- a/test/convert.cc +++ b/test/convert.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -23,6 +22,7 @@ #include #include #include +#include using namespace std; @@ -82,7 +82,7 @@ const char* extruRootPath = NULL; int ExtruRootId =0; bool found_bad_arg = false; -void getConfig(int argc, char** argv) { +void getConfig(int argc, char** argv, pcu::PCU *pcu_obj) { opterr = 0; @@ -120,18 +120,18 @@ void getConfig(int argc, char** argv) { gmi_native_path = optarg; break; case '?': - if (!PCU_Comm_Self()) + if (!pcu_obj->Self()) printf ("warning: skipping unrecognized option \'%s\'\n", argv[optind-1]); break; default: - if (!PCU_Comm_Self()) + if (!pcu_obj->Self()) printf("Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } } if(argc-optind != 3) { - if (!PCU_Comm_Self()) + if (!pcu_obj->Self()) printf("Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } @@ -139,7 +139,7 @@ void getConfig(int argc, char** argv) { gmi_path = argv[i++]; sms_path = argv[i++]; smb_path = argv[i++]; - if (!PCU_Comm_Self()) { + if (!pcu_obj->Self()) { printf ("fix_pyramids %d attach_order %d enable_log %d extruRootPath %s\n", should_fix_pyramids, should_attach_order, should_log, extruRootPath); printf ("native-model \'%s\' model \'%s\' simmetrix mesh \'%s\' output mesh \'%s\'\n", @@ -246,7 +246,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c int nvert=i; PList_delete(listV); - double coordNewPt[nvert][3]; + std::vector> coordNewPt(nvert,{0,0,0}); for(i=0; i< nvert ; i++) { int* markedData; if(!EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&markedData)){ // not sure about marked yet @@ -347,25 +347,25 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c int* vtxData = new int[1]; vtxData[0] = count2D; EN_attachDataPtr((pEntity)vrts[i],myFather,(void*)vtxData); - V_coord(vrts[i],coordNewPt[i]); + V_coord(vrts[i],coordNewPt[i].data()); fprintf ( fcr, "%.15E %.15E %d %d %d %d \n", coordNewPt[i][0],coordNewPt[i][1], vClassDim, foundESTag, foundETag, foundEETag ); } } - double coordFather[nvert][3]; + std::vector> coordFather(nvert,{0,0,0}); int fatherIds[4]; //store the ids of the fathers (vertices) on the root face for(i=0; i< nvert ; i++) { int* fatherIdPtr; const int exists = EN_getDataPtr((pEntity)vrts[i],myFather,(void**)&fatherIdPtr); if(!exists) { - if(!PCU_Comm_Self()) + if(!simApfMesh->getPCU()->Self()) fprintf(stderr, "Error: father id data pointer does not exist... exiting\n"); exit(EXIT_FAILURE); } assert(exists); fatherIds[i] = fatherIdPtr[0]; - V_coord(vrts[i],coordFather[i]); + V_coord(vrts[i],coordFather[i].data()); fprintf ( fcn, "%d ", fatherIds[i]); } fprintf ( fcn, "\n"); @@ -382,7 +382,7 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c faces=PList_new(); err = Extrusion_3DRegionsAndLayerFaces(region, regions, faces, 1); PList_delete(regions); // not used so delete - if(err!=1 && !PCU_Comm_Self()) + if(err!=1 && !simApfMesh->getPCU()->Self()) fprintf(stderr, "Extrusion_3DRegionsAndLayerFaces returned %d for err \n", err); // for each face in the returned list of faces @@ -432,7 +432,8 @@ void addFathersTag(pGModel simModel, pParMesh sim_mesh, apf::Mesh* simApfMesh, c int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); MS_init(); SimAdvMeshing_start(); //for fancy BL/extrusion queries @@ -440,12 +441,12 @@ int main(int argc, char** argv) Sim_readLicenseFile(NULL); SimPartitionedMesh_start(&argc,&argv); - getConfig(argc, argv); + getConfig(argc, argv, &pcu_obj); if( should_log ) Sim_logOn("convert.sim.log"); if (should_attach_order && should_fix_pyramids) { - if (!PCU_Comm_Self()) + if (!pcu_obj.Self()) std::cout << "disabling pyramid fix because --attach-order was given\n"; should_fix_pyramids = false; } @@ -457,7 +458,7 @@ int main(int argc, char** argv) gmi_model* mdl; if( gmi_native_path ) { - if (!PCU_Comm_Self()) + if (!pcu_obj.Self()) fprintf(stderr, "loading native model %s\n", gmi_native_path); mdl = gmi_sim_load(gmi_native_path,gmi_path); } else { @@ -483,24 +484,24 @@ int main(int argc, char** argv) assert(false); */ - double t0 = PCU_Time(); + double t0 = pcu::Time(); pParMesh sim_mesh = PM_load(sms_path, simModel, progress); - double t1 = PCU_Time(); - if(!PCU_Comm_Self()) + double t1 = pcu::Time(); + if(!pcu_obj.Self()) fprintf(stderr, "read and created the simmetrix mesh in %f seconds\n", t1-t0); - apf::Mesh* simApfMesh = apf::createMesh(sim_mesh); + apf::Mesh* simApfMesh = apf::createMesh(sim_mesh, &pcu_obj); addFathersTag(simModel, sim_mesh, simApfMesh, extruRootPath); - double t2 = PCU_Time(); - if(!PCU_Comm_Self()) + double t2 = pcu::Time(); + if(!simApfMesh->getPCU()->Self()) fprintf(stderr, "created the apf_sim mesh in %f seconds\n", t2-t1); if (should_attach_order) attachOrder(simApfMesh); apf::Mesh2* mesh = apf::createMdsMesh(mdl, simApfMesh); - double t3 = PCU_Time(); - if(!PCU_Comm_Self()) + double t3 = pcu::Time(); + if(!mesh->getPCU()->Self()) fprintf(stderr, "created the apf_mds mesh in %f seconds\n", t3-t2); apf::printStats(mesh); @@ -522,6 +523,6 @@ int main(int argc, char** argv) MS_exit(); if( should_log ) Sim_logOff(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/crack_test.cc b/test/crack_test.cc index 57764ec01..cbb7f1ef5 100644 --- a/test/crack_test.cc +++ b/test/crack_test.cc @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -23,17 +22,17 @@ void bCurver(const char* modelFile, const char* meshFile, - int order, const char* outputPrefix, + pcu::PCU *PCUObj, int order, const char* outputPrefix, int blendOrder = 0) { - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj); m->verify(); - if (PCU_Comm_Self() == 0) + if (m->getPCU()->Self() == 0) printf("attempting to curve the mesh to %d order Bezier\n", order); crv::BezierCurver bc(m, order, blendOrder); bc.run(); m->verify(); - if (PCU_Comm_Self() == 0) + if (m->getPCU()->Self() == 0) printf("succeeded!\n"); crv::writeCurvedVtuFiles(m, apf::Mesh::TRIANGLE, order + 2, outputPrefix); @@ -45,17 +44,19 @@ void bCurver(const char* modelFile, const char* meshFile, int main(int argc, char** argv) { + MPI_Init(&argc, &argv); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + if (argc < 2) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE: %s \n", argv[0]); } MPI_Finalize(); exit(EXIT_FAILURE); } - const char* modelFile = argv[1]; - MPI_Init(&argc, &argv); - PCU_Comm_Init(); + lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -68,7 +69,7 @@ int main(int argc, char** argv) gmi_register_mesh(); gmi_model* g = gmi_load(modelFile); - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(g, 2, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(g, 2, false, &PCUObj); std::vector verts; // vertex coordinates @@ -348,7 +349,7 @@ int main(int argc, char** argv) for (int i = 2; i < 7; i++) { char output[256]; snprintf(output,256,"crack_curved_to_order_%d", i); - bCurver(modelFile, "crack_linear.smb", i, output); + bCurver(modelFile, "crack_linear.smb", &PCUObj, i, output); } #ifdef HAVE_SIMMETRIX @@ -357,6 +358,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/create_mis.cc b/test/create_mis.cc index 40a43ddd6..bedc28f0d 100644 --- a/test/create_mis.cc +++ b/test/create_mis.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -11,10 +10,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -22,7 +22,7 @@ int main(int argc, char** argv) gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); //We will create a field with the coloring as the values on each element apf::Field* coloring = apf::createField(m,"colors",apf::SCALAR, @@ -60,7 +60,7 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/curve_to_bezier.cc b/test/curve_to_bezier.cc index 70f1f2576..fb9fadc0c 100644 --- a/test/curve_to_bezier.cc +++ b/test/curve_to_bezier.cc @@ -5,31 +5,32 @@ #include #include #include -#include #include #include #include #include #include + int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); MS_init(); SimModel_start(); Sim_readLicenseFile(0); gmi_sim_start(); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } int order = atoi(argv[3]); if(order < 1 || order > 6){ - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.Self() ) printf("Only 1st to 6th order supported\n"); MPI_Finalize(); exit(EXIT_FAILURE); @@ -37,7 +38,7 @@ int main(int argc, char** argv) gmi_register_mesh(); gmi_register_sim(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); crv::BezierCurver bc(m,order,0); bc.run(); @@ -53,6 +54,6 @@ int main(int argc, char** argv) Sim_unregisterAllKeys(); SimModel_stop(); MS_exit(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/curvetest.cc b/test/curvetest.cc index 3bf962ec2..2b4b51750 100644 --- a/test/curvetest.cc +++ b/test/curvetest.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -77,18 +76,18 @@ static void testElementSize(apf::Mesh* m) m->end(it); } - PCU_Add_Doubles(&(sizes[0]),3); - if(!PCU_Comm_Self()) + m->getPCU()->Add(&(sizes[0]),3); + if(!m->getPCU()->Self()) printf("Total sizes for order %d %f %f %f\n", m->getShape()->getOrder(), sizes[0],sizes[1],sizes[2]); } static void testInterpolating(const char* modelFile, const char* meshFile, - const int ne, const int nf) + const int ne, const int nf, pcu::PCU *PCUObj) { for(int order = 1; order <= 2; ++order){ - apf::Mesh2* m2 = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m2 = apf::loadMdsMesh(modelFile,meshFile,PCUObj); apf::changeMeshShape(m2,apf::getLagrange(order),true); crv::InterpolatingCurver ic(m2,order); ic.run(); @@ -103,7 +102,7 @@ static void testInterpolating(const char* modelFile, const char* meshFile, } static void testBezier(const char* modelFile, const char* meshFile, - const int ne, const int nf) + const int ne, const int nf, pcu::PCU *PCUObj) { apf::DynamicMatrix edgeErrors(ne,6); @@ -111,7 +110,7 @@ static void testBezier(const char* modelFile, const char* meshFile, // check blended shapes + interpolation error // interpolation error decreases as order is increased for(int order = 1; order <= 6; ++order){ - apf::Mesh2* m2 = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m2 = apf::loadMdsMesh(modelFile,meshFile,PCUObj); crv::BezierCurver bc(m2,order,2); bc.run(); testElementSize(m2); @@ -138,7 +137,7 @@ static void testBezier(const char* modelFile, const char* meshFile, } // check some refinement for(int order = 2; order <= 4; ++order){ - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj); crv::BezierCurver bc(m,order,0); bc.run(); Linear sf(m); @@ -153,7 +152,7 @@ static void testBezier(const char* modelFile, const char* meshFile, } // check some refinement for blended shapes for(int order = 2; order <= 4; ++order){ - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj); crv::BezierCurver bc(m,order,1); bc.run(); Linear sf(m); @@ -170,10 +169,10 @@ static void testBezier(const char* modelFile, const char* meshFile, } static void testGregory(const char* modelFile, const char* meshFile, - const int ne, const int nf) + const int ne, const int nf, pcu::PCU *PCUObj) { for(int order = 0; order <= 2; ++order){ - apf::Mesh2* m2 = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m2 = apf::loadMdsMesh(modelFile,meshFile,PCUObj); crv::GregoryCurver gc(m2,4,order); gc.run(); testElementSize(m2); @@ -192,24 +191,25 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); MS_init(); SimModel_start(); Sim_readLicenseFile(0); gmi_sim_start(); gmi_register_sim(); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); int ne = m->count(1); int nf = m->count(2); m->destroyNative(); apf::destroyMesh(m); - testInterpolating(modelFile,meshFile,ne,nf); - testBezier(modelFile,meshFile,ne,nf); - testGregory(modelFile,meshFile,ne,nf); + testInterpolating(modelFile,meshFile,ne,nf,&PCUObj); + testBezier(modelFile,meshFile,ne,nf,&PCUObj); + testGregory(modelFile,meshFile,ne,nf,&PCUObj); - PCU_Comm_Free(); + } gmi_sim_stop(); Sim_unregisterAllKeys(); SimModel_stop(); diff --git a/test/degenerateSurfs.cc b/test/degenerateSurfs.cc index 730e8cbf7..71ece5203 100644 --- a/test/degenerateSurfs.cc +++ b/test/degenerateSurfs.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -19,10 +18,10 @@ const char* meshFile = 0; const char* outFile = 0; int level = 1; -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -37,7 +36,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==5); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -47,8 +47,8 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + getConfig(argc,argv,&PCUObj); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); int order = 2; @@ -71,7 +71,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/describe.cc b/test/describe.cc index 07f99a4a9..4ed8b59eb 100644 --- a/test/describe.cc +++ b/test/describe.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -18,21 +17,23 @@ #endif #include -static void print_stats(const char* name, double value) + +static void print_stats(const char* name, double value, pcu::PCU *PCUObj) + { double min, max, avg; - min = PCU_Min_Double(value); - max = PCU_Max_Double(value); - avg = PCU_Add_Double(value); - avg /= PCU_Comm_Peers(); + min = PCUObj->Min(value); + max = PCUObj->Max(value); + avg = PCUObj->Add(value); + avg /= PCUObj->Peers(); double imb = max / avg; - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf("%s: min %f max %f avg %f imb %f\n", name, min, max, avg, imb); } static void list_tags(apf::Mesh* m) { - if (PCU_Comm_Self()) + if (m->getPCU()->Self()) return; apf::DynamicArray tags; m->getTags(tags); @@ -44,7 +45,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -54,10 +56,10 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - print_stats("kernel used before", PCU_GetMem()); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + print_stats("kernal used before", pcu::GetMem(), &pcu_obj); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); m->verify(); - print_stats("kernel heap", PCU_GetMem()); + print_stats("kernel heap", pcu::GetMem(), &pcu_obj); Parma_PrintPtnStats(m, ""); list_tags(m); m->destroyNative(); @@ -68,6 +70,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/dg_ma_test.cc b/test/dg_ma_test.cc index 188592c6a..5e2b7b50a 100644 --- a/test/dg_ma_test.cc +++ b/test/dg_ma_test.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -41,7 +40,8 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -51,11 +51,11 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); m->verify(); Linear sf(m); ma::Input* in = ma::makeAdvanced(ma::configure(m, &sf)); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) printf("Matched mesh: disabling" " snapping, and shape correction,\n"); in->shouldSnap = false; @@ -76,7 +76,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/elmBalance.cc b/test/elmBalance.cc index 059c18657..be0d2a2bf 100644 --- a/test/elmBalance.cc +++ b/test/elmBalance.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -29,10 +28,11 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -45,10 +45,10 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); double imbalance[4]; Parma_GetEntImbalance(m,&imbalance); - if(!PCU_Comm_Self()) + if(!m->getPCU()->Self()) fprintf(stdout, "imbalance %.3f %.3f %.3f %.3f\n", imbalance[0], imbalance[1], imbalance[2], imbalance[3]); apf::MeshTag* weights = setWeights(m); @@ -67,6 +67,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/embedded_edges.cc b/test/embedded_edges.cc index be0948569..fcfa4b565 100644 --- a/test/embedded_edges.cc +++ b/test/embedded_edges.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -17,7 +16,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==2); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -28,7 +28,7 @@ int main(int argc, char** argv) int nverts; gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::loadMdsMesh(model, argv[1]); + apf::Mesh2* m = apf::loadMdsMesh(model, argv[1], &PCUObj); apf::deriveMdsModel(m); int dim = m->getDimension(); extractCoords(m, coords, nverts); @@ -40,7 +40,7 @@ int main(int argc, char** argv) std::cout<typeDimension[apf::Mesh::EDGE]<destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/extrude.cc b/test/extrude.cc index f3c5fc1d6..85e3aeb59 100644 --- a/test/extrude.cc +++ b/test/extrude.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -11,16 +10,17 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); ma::ModelExtrusions extrusions; extrusions.push_back(ma::ModelExtrusion( m->findModelEntity(1, 2), @@ -39,6 +39,6 @@ int main(int argc, char** argv) m->writeNative(argv[4]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/fieldReduce.cc b/test/fieldReduce.cc index df70a5409..4bc94ac65 100644 --- a/test/fieldReduce.cc +++ b/test/fieldReduce.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -147,10 +146,10 @@ void freeMesh(apf::Mesh* m) apf::destroyMesh(m); } -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 3 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -166,10 +165,12 @@ void getConfig(int argc, char** argv) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + bool failflag = false; + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - MPI_Comm_rank(PCU_Get_Comm(), &myrank); - MPI_Comm_size(PCU_Get_Comm(), &commsize); + MPI_Comm_rank(PCUObj.GetMPIComm(), &myrank); + MPI_Comm_size(PCUObj.GetMPIComm(), &commsize); #ifdef HAVE_SIMMETRIX MS_init(); SimModel_start(); @@ -178,13 +179,13 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv); + getConfig(argc,argv,&PCUObj); gmi_model* g = 0; g = gmi_load(modelFile); apf::Mesh2* m = 0; - m = apf::loadMdsMesh(g, meshFile); + m = apf::loadMdsMesh(g, meshFile, &PCUObj); - bool failflag = false; + for (int i=0; i < 3; ++i) failflag = failflag || testReduce(m, i); @@ -195,9 +196,10 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); return failflag; + } diff --git a/test/field_io.cc b/test/field_io.cc index d1514c1db..44c2c8305 100644 --- a/test/field_io.cc +++ b/test/field_io.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -11,11 +10,12 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 3); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); { - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], &PCUObj); apf::Field* f = apf::createLagrangeField(m, "foo", apf::VECTOR, 1); apf::MeshIterator* it = m->begin(0); apf::MeshEntity* vert; @@ -28,7 +28,7 @@ int main(int argc, char** argv) apf::destroyMesh(m); } { - apf::Mesh2* m = apf::loadMdsMesh(argv[1], "tmp.smb"); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], "tmp.smb", &PCUObj); apf::Field* f = m->findField("foo"); PCU_ALWAYS_ASSERT(f); PCU_ALWAYS_ASSERT(apf::VECTOR == apf::getValueType(f)); @@ -46,6 +46,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); } - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/fixDisconnected.cc b/test/fixDisconnected.cc index 5d7174c49..5901e84b5 100644 --- a/test/fixDisconnected.cc +++ b/test/fixDisconnected.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -12,22 +11,23 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); Parma_PrintPtnStats(m, "initial"); Parma_ProcessDisconnectedParts(m); Parma_PrintPtnStats(m, "final"); m->writeNative(argv[3]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/fixlayer.cc b/test/fixlayer.cc index 0827bc025..f018e51c0 100644 --- a/test/fixlayer.cc +++ b/test/fixlayer.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -16,7 +15,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -26,7 +26,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(argv[1], argv[2]); + ma::Mesh* m = apf::loadMdsMesh(argv[1], argv[2], &pcu_obj); ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m)); in->shouldCleanupLayer = true; ma::adapt(in); @@ -39,7 +39,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/fixshape.cc b/test/fixshape.cc index 899e1c0e4..fc32ab791 100644 --- a/test/fixshape.cc +++ b/test/fixshape.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -16,7 +15,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -26,7 +26,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2]); + ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m)); in->shouldFixShape = true; ma::adapt(in); @@ -39,7 +39,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/fusion.cc b/test/fusion.cc index 7da4c72f2..f9fc9e216 100644 --- a/test/fusion.cc +++ b/test/fusion.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -142,12 +141,12 @@ struct GroupCode : public Parma_GroupCode void run(int group) { if (group == 0) { - mesh = apf::loadMdsMesh(model, meshFile); + mesh = apf::loadMdsMesh(model, meshFile, PCUObj.get()); mesh->verify(); testIndexing(mesh); fusionAdapt(mesh); } else { - mesh = apf::makeEmptyMdsMesh(model, 2, false); + mesh = apf::makeEmptyMdsMesh(model, 2, false, PCUObj.get()); } } }; @@ -156,17 +155,21 @@ int main( int argc, char* argv[]) { PCU_ALWAYS_ASSERT(argc==2); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); GroupCode code; code.model = makeModel(); code.meshFile = argv[1]; - apf::Unmodulo outMap(PCU_Comm_Self(), 2); - Parma_SplitPartition(NULL, 2, code); + apf::Unmodulo outMap(pcu_obj.Self(), 2); + Parma_SplitPartition(nullptr, 2, code, &pcu_obj); + //Have to call switchPCU here because the mesh needed to be made inside GroupCode run() + //and inside parma_group.cc runInGroups() we set code.PCUObj to groupedPCU + code.mesh->switchPCU(&pcu_obj); apf::remapPartition(code.mesh, outMap); code.mesh->verify(); code.mesh->destroyNative(); apf::destroyMesh(code.mesh); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/fusion2.cc b/test/fusion2.cc index eef2b6c10..d252af990 100644 --- a/test/fusion2.cc +++ b/test/fusion2.cc @@ -2,17 +2,16 @@ #include #include #include -#include #include #include #include #include -static apf::Mesh2* makeEmptyMesh() +static apf::Mesh2* makeEmptyMesh(pcu::PCU *PCUObj) { /* dont use null model in production */ gmi_model* g = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(g, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(g, 2, false, PCUObj); /* both planes create the field at the beginning */ apf::createPackedField(m, "fusion", 6); return m; @@ -68,7 +67,7 @@ static void backToTags(apf::Mesh2* m) static void connectPlanes(apf::Mesh2* m) { /* dont use any of this function in production. */ - int peer = 1 - PCU_Comm_Self(); + int peer = 1 - m->getPCU()->Self(); for (int i = 0; i < 3; ++i) { apf::MeshEntity* v = apf::getMdsEntity(m, 0, i); m->addRemote(v, peer, v); //pointers aren't usually equal @@ -92,7 +91,6 @@ struct GroupCode : public Parma_GroupCode apf::Mesh2* mesh; void run(int group) { - mesh = ::makeEmptyMesh(); if (group == 0) { addOneTri(mesh); setValues(mesh); @@ -108,7 +106,7 @@ struct GroupCode : public Parma_GroupCode static void copyPlane(apf::Mesh2* m) { /* this mimics the copy entity construction */ - if (PCU_Comm_Self() == 1) + if (m->getPCU()->Self() == 1) addOneTri(m); /* this connects the two one-triangle planes together */ connectPlanes(m); @@ -140,17 +138,17 @@ static void globalCode(apf::Mesh2* m) int main( int argc, char* argv[]) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT(PCU_Comm_Peers() == 2); + PCU_ALWAYS_ASSERT(PCUObj.Peers() == 2); gmi_register_null(); GroupCode code; + code.mesh = makeEmptyMesh(&PCUObj); int const groupSize = 1; - apf::Unmodulo outMap(PCU_Comm_Self(), groupSize); - Parma_SplitPartition(NULL, groupSize, code); - /* update mesh for leaving groups */ - apf::remapPartition(code.mesh, outMap); + apf::Unmodulo outMap(PCUObj.Self(), groupSize); + Parma_SplitPartition(code.mesh, groupSize, code, &PCUObj); globalCode(code.mesh); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/fusion3.cc b/test/fusion3.cc index a8df775e4..ee3e562c1 100644 --- a/test/fusion3.cc +++ b/test/fusion3.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -283,11 +282,12 @@ int main(int argc, char * argv[]) { PCU_ALWAYS_ASSERT(argc==2); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_model* model = makeModel(); gmi_write_dmg(model, "made.dmg"); - apf::Mesh2* mesh=apf::loadMdsMesh(model, argv[1]); + apf::Mesh2* mesh=apf::loadMdsMesh(model, argv[1], &PCUObj); mesh->verify(); Vortex sfv(mesh, center, modelLen); const ma::Input* in = ma::configure(mesh,&sfv); @@ -296,4 +296,7 @@ int main(int argc, char * argv[]) apf::writeVtkFiles("adapted",mesh); //clean data // to do + apf::destroyMesh(mesh); + } + MPI_Finalize(); } diff --git a/test/gap.cc b/test/gap.cc index 2b1e46af5..36554ffd7 100644 --- a/test/gap.cc +++ b/test/gap.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -30,10 +29,11 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 5); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -47,7 +47,7 @@ int main(int argc, char** argv) #endif gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); Parma_PrintPtnStats(m, "initial"); apf::MeshTag* weights = setWeights(m); int verbose = 2; // set to 1 to silence the 'endStep' stats @@ -68,6 +68,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/generate.cc b/test/generate.cc index 473716a88..3f0ad6768 100644 --- a/test/generate.cc +++ b/test/generate.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -19,6 +18,7 @@ #include #include #include +#include #include #include @@ -41,6 +41,7 @@ #include "SimAttribute.h" #include "ModelTypes.h" +static const pcu::PCU* globalPCU = nullptr; namespace { @@ -59,11 +60,11 @@ void messageHandler(int type, const char* msg) { switch (type) { case Sim_WarningMsg: - if(!PCU_Comm_Self()) + if(!globalPCU->Self()) fprintf(stdout, "Warning SimModeler %s\n", msg); break; case Sim_ErrorMsg: - if(!PCU_Comm_Self()) + if(!globalPCU->Self()) fprintf(stdout, "Error SimModeler %s ... exiting\n", msg); MPI_Finalize(); exit(EXIT_SUCCESS); @@ -74,13 +75,13 @@ void messageHandler(int type, const char* msg) } } -pParMesh generate(pGModel mdl, std::string meshCaseName) { +pParMesh generate(pGModel mdl, std::string meshCaseName, pcu::PCU *PCUObj) { #if SIMMODSUITE_MAJOR_VERSION <= 2024 && SIMMODSUITE_MINOR_VERSION < 240219 pAManager attmngr = GM_attManager(mdl); #else pAManager attmngr = GM_attManager(mdl,true); #endif - if(0==PCU_Comm_Self()) + if(0==PCUObj->Self()) fprintf(stdout, "Loading mesh case %s...\n", meshCaseName.c_str()); pACase mcaseFile = AMAN_findCase(attmngr, meshCaseName.c_str()); PCU_ALWAYS_ASSERT(mcaseFile); @@ -106,16 +107,16 @@ pParMesh generate(pGModel mdl, std::string meshCaseName) { if( !disable_surface ) { const double stime = MPI_Wtime(); - if(0==PCU_Comm_Self()) { + if(0==PCUObj->Self()) { printf("Meshing surface..."); fflush(stdout); } pSurfaceMesher surfaceMesher = SurfaceMesher_new(mcase, pmesh); SurfaceMesher_execute(surfaceMesher, NULL); SurfaceMesher_delete(surfaceMesher); - if(0==PCU_Comm_Self()) + if(0==PCUObj->Self()) printf(" %f seconds\n", MPI_Wtime()-stime); if( ! surfaceMeshFile.empty() ) { - if(0==PCU_Comm_Self()) + if(0==PCUObj->Self()) printf(" writing surface mesh %s\n", surfaceMeshFile.c_str()); PM_write(pmesh, surfaceMeshFile.c_str(), NULL); } @@ -123,20 +124,20 @@ pParMesh generate(pGModel mdl, std::string meshCaseName) { if( !disable_volume ) { const double vtime = MPI_Wtime(); - if(0==PCU_Comm_Self()) { + if(0==PCUObj->Self()) { printf("Meshing volume..."); fflush(stdout); } pVolumeMesher volumeMesher = VolumeMesher_new(mcase, pmesh); VolumeMesher_execute(volumeMesher, NULL); VolumeMesher_delete(volumeMesher); - if(0==PCU_Comm_Self()) + if(0==PCUObj->Self()) printf(" %f seconds\n", MPI_Wtime()-vtime); } return pmesh; } -void getConfig(int argc, char** argv) { +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { opterr = 0; static struct option long_opts[] = { @@ -169,7 +170,7 @@ void getConfig(int argc, char** argv) { if (c == -1) break; //end of options switch (c) { case 0: // enable-log|disable-volume|disable-surf - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf ("read arg %d\n", c); break; case 'n': @@ -179,11 +180,11 @@ void getConfig(int argc, char** argv) { surfaceMeshFile = std::string(optarg); break; case '?': - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf ("warning: skipping unrecognized option\n"); break; default: - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf("Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } @@ -191,7 +192,7 @@ void getConfig(int argc, char** argv) { if(argc-optind != 2) { - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf("Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } @@ -200,7 +201,7 @@ void getConfig(int argc, char** argv) { outMeshFile = caseName = std::string(argv[i++]); outMeshFile.append("/"); - if (!PCU_Comm_Self()) { + if (!PCUObj->Self()) { std::cout << "enable_log " << should_log << " disable_surface " << disable_surface << " disable_volume " << disable_volume << @@ -254,7 +255,7 @@ bool hasExtension(std::string s, std::string ext) { } #endif -pNativeModel loadNativeModel() { +pNativeModel loadNativeModel(pcu::PCU *PCUObj) { enum { TEXT_FORMAT = 0 }; pNativeModel nm = 0; if ( nativeModelFile.empty() ) { @@ -262,18 +263,18 @@ pNativeModel loadNativeModel() { #ifdef SIM_ACIS } else if (hasExtension(nativeModelFile, "sat")) { nm = AcisNM_createFromFile(nativeModelFile.c_str(), TEXT_FORMAT); - if(!PCU_Comm_Self()) + if(!PCUObj->Self()) printf("loaded acis native model\n"); #endif #ifdef SIM_PARASOLID } else if (hasExtension(nativeModelFile, "xmt_txt") || hasExtension(nativeModelFile, "x_t") ) { nm = ParasolidNM_createFromFile(nativeModelFile.c_str(), TEXT_FORMAT); - if(!PCU_Comm_Self()) + if(!PCUObj->Self()) printf("loaded parasolid native model\n"); #endif } else { - if(!PCU_Comm_Self()) + if(!PCUObj->Self()) printf("native model file has bad extension"); exit(EXIT_FAILURE); } @@ -320,27 +321,29 @@ void simStop() { int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + globalPCU = &PCUObj; lion_set_verbosity(1); - PCU_Protect(); - getConfig(argc,argv); + pcu::Protect(); + getConfig(argc,argv,&PCUObj); if (should_attach_order && should_fix_pyramids) { - if (!PCU_Comm_Self()) + if (!PCUObj.Self()) std::cout << "disabling pyramid fix because --attach-order was given\n"; should_fix_pyramids = 0; } simStart(); - pNativeModel nm = loadNativeModel(); + pNativeModel nm = loadNativeModel(&PCUObj); pGModel simModel = GM_load(modelFile.c_str(), nm, NULL); const double t0 = MPI_Wtime(); - pParMesh sim_mesh = generate(simModel, caseName); + pParMesh sim_mesh = generate(simModel, caseName, &PCUObj); const double t1 = MPI_Wtime(); - if(!PCU_Comm_Self()) + if(!PCUObj.Self()) printf("Mesh generated in %f seconds\n", t1-t0); - apf::Mesh* simApfMesh = apf::createMesh(sim_mesh); + apf::Mesh* simApfMesh = apf::createMesh(sim_mesh, &PCUObj); if (should_attach_order) attachOrder(simApfMesh); gmi_register_sim(); @@ -357,6 +360,6 @@ int main(int argc, char** argv) simStop(); Sim_logOff(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ghost.cc b/test/ghost.cc index d225f2221..190feb5eb 100644 --- a/test/ghost.cc +++ b/test/ghost.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -18,10 +17,10 @@ namespace { apf::destroyMesh(m); } - void getConfig(int argc, char** argv) + void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -78,16 +77,17 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + getConfig(argc,argv,&PCUObj); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); apf::MeshTag* weights = applyFun3dWeight(m); runParma(m,weights); m->destroyTag(weights); Parma_WriteVtxPtn(m,argv[3]); freeMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ghostEdge.cc b/test/ghostEdge.cc index fbacc304a..6f4d3ae40 100644 --- a/test/ghostEdge.cc +++ b/test/ghostEdge.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -51,14 +50,15 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); runParma(m); m->writeNative(argv[3]); freeMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ghostMPAS.cc b/test/ghostMPAS.cc index 8e1ea1715..8af944c6a 100644 --- a/test/ghostMPAS.cc +++ b/test/ghostMPAS.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -53,14 +52,15 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); runParma(m); m->writeNative(argv[3]); freeMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/gmsh.cc b/test/gmsh.cc index b2f3911ec..2161ec764 100644 --- a/test/gmsh.cc +++ b/test/gmsh.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -11,10 +10,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n" "The input .msh and output .smb file names are required. \n" "If 'none' is specified as the input model file name then \n" @@ -39,22 +39,22 @@ int main(int argc, char** argv) apf::Mesh2* m = NULL; if (gmshVersion == 2) { if (model.compare("none") == 0) { - m = apf::loadMdsFromGmsh(gmi_load(".null"), gmsh.c_str()); + m = apf::loadMdsFromGmsh(gmi_load(".null"), gmsh.c_str(), &PCUObj); apf::deriveMdsModel(m); gmi_write_dmg(m->getModel(),outModel.c_str()); } else { - m = apf::loadMdsFromGmsh(gmi_load(model.c_str()), gmsh.c_str()); + m = apf::loadMdsFromGmsh(gmi_load(model.c_str()), gmsh.c_str(), &PCUObj); } } else if (gmshVersion == 4) { if (model.compare("none") == 0) { - m = apf::loadMdsDmgFromGmsh(outModel.c_str(), gmsh.c_str()); + m = apf::loadMdsDmgFromGmsh(outModel.c_str(), gmsh.c_str(), &PCUObj); } } m->verify(); m->writeNative(outMesh.c_str()); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/graphdist.cc b/test/graphdist.cc index 98fd857a9..3d9e96041 100644 --- a/test/graphdist.cc +++ b/test/graphdist.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -11,16 +10,17 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); dsp::Boundary moving; moving.insert(m->findModelEntity(2, 57)); moving.insert(m->findModelEntity(2, 62)); @@ -31,6 +31,6 @@ int main(int argc, char** argv) apf::writeVtkFiles(argv[3], m); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/hierarchic.cc b/test/hierarchic.cc index 50e5bf24d..26bccc7de 100644 --- a/test/hierarchic.cc +++ b/test/hierarchic.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -194,17 +193,18 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT(! PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(! pcu_obj.Self()); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], &pcu_obj); apf::reorderMdsMesh(m); m->verify(); int p_order = atoi(argv[3]); test(m, p_order); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/highOrderSizeFields.cc b/test/highOrderSizeFields.cc index 7cbbe34fc..db6a71ffb 100644 --- a/test/highOrderSizeFields.cc +++ b/test/highOrderSizeFields.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -31,12 +30,14 @@ void computeSizesFrames( void testAdapt( const char* model, const char* mesh, - int order); + int order, + pcu::PCU *PCUObj); int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX @@ -49,19 +50,19 @@ int main(int argc, char** argv) gmi_register_mesh(); if (argc != 3) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.Self()) std::cerr <<"usage: " << argv[0] << " \n"; return EXIT_FAILURE; } for (int order = 3; order < 5; order++) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.Self()) lion_oprint(1, "Testing aniso adapt w/ sizefield order %d\n", order); - testAdapt(argv[1], argv[2], order); + testAdapt(argv[1], argv[2], order, &PCUObj); } - PCU_Comm_Free(); + } #ifdef HAVE_SIMMETRIX gmi_sim_stop(); Sim_unregisterAllKeys(); @@ -113,10 +114,11 @@ void computeSizesFrames( void testAdapt( const char* model, const char* mesh, - int order) + int order, + pcu::PCU *PCUObj) { gmi_model* g = gmi_load(model); - apf::Mesh2* m = apf::loadMdsMesh(g,mesh); + apf::Mesh2* m = apf::loadMdsMesh(g,mesh,PCUObj); m->verify(); const ma::Input* in = ma::configureUniformRefine(m, 1, 0); diff --git a/test/highOrderSolutionTransfer.cc b/test/highOrderSolutionTransfer.cc index bba9616cd..57eb318d5 100644 --- a/test/highOrderSolutionTransfer.cc +++ b/test/highOrderSolutionTransfer.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -33,6 +32,7 @@ double testH1Field( void testCurveAdapt( const char* modelFile, const char* meshFile, + pcu::PCU *PCUObj, const int mesh_order, const int exact_order, const int field_order); @@ -42,7 +42,8 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -62,28 +63,28 @@ int main(int argc, char** argv) */ // linear adapt - testCurveAdapt(modelFile, meshFile, + testCurveAdapt(modelFile, meshFile, &PCUObj, 1 /*mesh_order*/, 2 /*exact_order*/, 2 /*field_order*/); // quadratic adapts - testCurveAdapt(modelFile, meshFile, + testCurveAdapt(modelFile, meshFile, &PCUObj, 2 /*mesh_order*/, 2 /*exact_order*/, 4 /*field_order*/); - testCurveAdapt(modelFile, meshFile, + testCurveAdapt(modelFile, meshFile, &PCUObj, 2 /*mesh_order*/, 3 /*exact_order*/, 6 /*field_order*/); // cubic adapt - testCurveAdapt(modelFile, meshFile, + testCurveAdapt(modelFile, meshFile, &PCUObj, 3 /*mesh_order*/, 2 /*exact_order*/, 6 /*field_order*/); - PCU_Comm_Free(); + } #ifdef HAVE_SIMMETRIX gmi_sim_stop(); Sim_unregisterAllKeys(); @@ -193,12 +194,13 @@ double testH1Field( void testCurveAdapt( const char* modelFile, const char* meshFile, + pcu::PCU *PCUObj, const int mesh_order, const int exact_order, const int field_order) { - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,PCUObj); m->verify(); if (mesh_order > 1) { diff --git a/test/icesheet.cc b/test/icesheet.cc index ac2decfb7..a05b4061d 100644 --- a/test/icesheet.cc +++ b/test/icesheet.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -483,7 +482,8 @@ int main(int argc, char** argv) } MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); gmi_register_null(); @@ -494,7 +494,7 @@ int main(int argc, char** argv) readMesh(argv[1],m); const int dim = 3; - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, false); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, dim, false, &PCUObj); apf::GlobalToVert outMap; apf::construct(mesh, m.elements, m.numElms, m.elementType, outMap); delete [] m.elements; @@ -524,6 +524,6 @@ int main(int argc, char** argv) mesh->destroyNative(); apf::destroyMesh(mesh); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/inClosureOf_test.cc b/test/inClosureOf_test.cc index 7b673dee3..f76448979 100644 --- a/test/inClosureOf_test.cc +++ b/test/inClosureOf_test.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -20,7 +19,8 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -87,7 +87,7 @@ int main(int argc, char** argv) gmi_destroy(model); // deleting the model - PCU_Comm_Free(); + } #ifdef HAVE_SIMMETRIX gmi_sim_stop(); Sim_unregisterAllKeys(); diff --git a/test/intrude.cc b/test/intrude.cc index f5bc2f893..bf1fbe332 100644 --- a/test/intrude.cc +++ b/test/intrude.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -11,16 +10,17 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); ma::ModelExtrusions extrusions; extrusions.push_back(ma::ModelExtrusion( m->findModelEntity(1, 2), @@ -36,11 +36,11 @@ int main(int argc, char** argv) m->findModelEntity(0, 1))); size_t nlayers; ma::intrude(m, extrusions, &nlayers); - if (!PCU_Comm_Self()) + if (!m->getPCU()->Self()) std::cout << "counted " << nlayers << " layers\n"; m->writeNative(argv[3]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/loadPart.cc b/test/loadPart.cc index 902ed0dbf..32d4067f7 100644 --- a/test/loadPart.cc +++ b/test/loadPart.cc @@ -3,19 +3,20 @@ #include #include #include -#include #include #include #include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT(PCU_Comm_Peers() == 1); + PCU_ALWAYS_ASSERT(PCUObj.Peers() == 1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Load a single part from a partitioned mesh and " "write it as a serial part.\n" "Usage: %s \n", argv[0]); @@ -25,13 +26,13 @@ int main(int argc, char** argv) gmi_register_null(); gmi_register_mesh(); gmi_model* g = gmi_load(".null"); - apf::Mesh2* m = apf::loadMdsPart(g, argv[1]); + apf::Mesh2* m = apf::loadMdsPart(g, argv[1], &PCUObj); apf::deriveMdsModel(m); gmi_write_dmg(g, argv[3]); m->writeNative(argv[2]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ma_insphere.cc b/test/ma_insphere.cc index 1fd0dc739..4b990a705 100644 --- a/test/ma_insphere.cc +++ b/test/ma_insphere.cc @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include @@ -30,7 +29,8 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(apf::getDeterminant(matrix) == -1485); // Test insphere (create a mesh with one tet) - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); apf::Vector3 a(0, 0, 0); apf::Vector3 b(-6, 0, 0); @@ -39,7 +39,7 @@ int main(int argc, char** argv) gmi_register_null(); gmi_model* model = gmi_load(".null"); - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, 3, true); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, 3, true, &PCUObj); apf::ModelEntity* m = mesh->findModelEntity(0, 0); apf::MeshEntity* v[4]; for (int i=0; i<4; i++) { @@ -57,6 +57,6 @@ int main(int argc, char** argv) mesh->destroyNative(); apf::destroyMesh(mesh); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ma_test.cc b/test/ma_test.cc index 8d4558f24..86100546b 100644 --- a/test/ma_test.cc +++ b/test/ma_test.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -45,7 +44,8 @@ int main(int argc, char** argv) const char* layerTagString = (argc==4) ? argv[3] : ""; const double adaptRefineFactor = (argc==5) ? atoi(argv[4]) : 3; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -55,7 +55,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); m->verify(); Linear sf(m,adaptRefineFactor); ma::Input* in = ma::makeAdvanced(ma::configure(m, &sf)); @@ -80,7 +80,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ma_test_analytic_model.cc b/test/ma_test_analytic_model.cc index 6d5d45635..1a03f233d 100644 --- a/test/ma_test_analytic_model.cc +++ b/test/ma_test_analytic_model.cc @@ -9,7 +9,6 @@ #include #include #include -#include #include #include @@ -65,10 +64,10 @@ gmi_model* makeSphere() return model; } -apf::Mesh2* createSphereMesh() +apf::Mesh2* createSphereMesh(pcu::PCU *PCUObj) { gmi_model* model = makeSphere(); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::MeshEntity* allVs[5]; apf::Vector3 p0( cos(0.), sin(0.), 0.); apf::Vector3 p1( cos(2.*pi/3.), sin(2.*pi/3.), 0.); @@ -171,10 +170,11 @@ apf::Mesh2* createSphereMesh() int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - apf::Mesh2* m = createSphereMesh(); + apf::Mesh2* m = createSphereMesh(&PCUObj); m->verify(); apf::writeVtkFiles("initial_mesh_on_analytic_model", m); @@ -186,7 +186,6 @@ int main(int argc, char** argv) m->destroyNative(); apf::destroyMesh(m); - - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/makeAllCavities.cc b/test/makeAllCavities.cc index a1ee74ab9..df5692c48 100644 --- a/test/makeAllCavities.cc +++ b/test/makeAllCavities.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -63,8 +62,9 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); - if (PCU_Comm_Peers() > 1) { + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + if (PCUObj.Peers() > 1) { printf("%s should only be used for serial (single part) meshes!\n", argv[0]); printf("use the serialize utility to get a serial mesh, and retry!\n"); MPI_Finalize(); @@ -111,7 +111,7 @@ int main(int argc, char** argv) // load the mesh and check if the tag exists on the mesh - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); if (mode.compare(std::string("aa")) == 0) tag = tagMesh(m, -1, 1); @@ -253,7 +253,7 @@ int main(int argc, char** argv) MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -292,7 +292,7 @@ static apf::Mesh2* makePoint( apf::MeshEntity* e) { PCU_ALWAYS_ASSERT(m->getType(e) == apf::Mesh::VERTEX); - apf::Mesh2* sphMesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false); + apf::Mesh2* sphMesh = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false, m->getPCU()); double xrange[2] = {1.e16, -1.e16}; double yrange[2] = {1.e16, -1.e16}; double zrange[2] = {1.e16, -1.e16}; @@ -403,8 +403,8 @@ static void makeEntMeshes( } if (m->getType(e) == apf::Mesh::EDGE) { - entMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false); - entMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false); + entMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false, m->getPCU()); + entMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), 1, false, m->getPCU()); apf::MeshEntity* vs[2]; m->getDownward(e, 0, vs); apf::Vector3 p[2]; @@ -443,8 +443,8 @@ static void makeEntMeshes( } if (m->getType(e) == apf::Mesh::TRIANGLE) { - entMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false); - entMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false); + entMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false, m->getPCU()); + entMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), 2, false, m->getPCU()); apf::MeshEntity* downverts[3]; apf::MeshEntity* downedges[3]; m->getDownward(e, 0, downverts); @@ -657,8 +657,8 @@ static void makeCavityMeshes( // we do this in a bottom up fashion, ie vets first then edges, faces and tets /* cavityMeshLinear = apf::makeEmptyMdsMesh(m->getModel(), dim, false); */ /* cavityMeshCurved = apf::makeEmptyMdsMesh(m->getModel(), dim, false); */ - cavityMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), dim, false); - cavityMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), dim, false); + cavityMeshLinear = apf::makeEmptyMdsMesh(gmi_load(".null"), dim, false, m->getPCU()); + cavityMeshCurved = apf::makeEmptyMdsMesh(gmi_load(".null"), dim, false, m->getPCU()); // verts for (int i = 0; i < (int) icavity0.size(); i++) { diff --git a/test/matchedNodeElmReader.cc b/test/matchedNodeElmReader.cc index 7175b531a..a135f0703 100644 --- a/test/matchedNodeElmReader.cc +++ b/test/matchedNodeElmReader.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -158,7 +157,7 @@ void setVtxClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxC mesh->end(it); } -void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh,apf::MeshTag* vtxClass) { +void setEdgeClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* vtxClass) { (void)model; (void)mesh; (void)vtxClass; @@ -424,11 +423,11 @@ void setClassification(gmi_model* model, apf::Mesh2* mesh, apf::MeshTag* t) { } - + void getLocalRange(apf::Gid total, int& local, - apf::Gid& first, apf::Gid& last) { - const int self = PCU_Comm_Self(); - const int peers = PCU_Comm_Peers(); + apf::Gid& first, apf::Gid& last, pcu::PCU *PCUObj) { + const int self = PCUObj->Self(); + const int peers = PCUObj->Peers(); local = total/peers; if( self == peers-1 ) { //last rank apf::Gid lp=local*peers; @@ -438,7 +437,7 @@ void getLocalRange(apf::Gid total, int& local, local += lpd; } } - first = PCU_Exscan_Long(local); + first = PCUObj->Exscan(local); last = first+local; } @@ -529,9 +528,9 @@ void readSolution(FILE* f, int& localnumvtx, double** solution) { } } -void readMatches(FILE* f, apf::Gid numvtx, int localnumvtx, apf::Gid** matches) { +void readMatches(FILE* f, apf::Gid numvtx, int localnumvtx, apf::Gid** matches, pcu::PCU *PCUObj) { fprintf(stderr, "%d readMatches numvtx %ld localnumvtx %d \n", - PCU_Comm_Self(), numvtx, localnumvtx); + PCUObj->Self(), numvtx, localnumvtx); *matches = new apf::Gid[localnumvtx]; rewind(f); apf::Gid matchedVtx; @@ -610,9 +609,9 @@ fh = header file (there is only one for all processes), containing: ... for as many topos as are in Part 0 Repeat the above bock for each part. **/ -std::vector readHeader(std::ifstream& fh) { +std::vector readHeader(std::ifstream& fh, pcu::PCU *PCUObj) { rewindStream(fh); - const int self = PCU_Comm_Self();; + const int self = PCUObj->Self();; bool ret = seekPart(fh, std::to_string(self)); PCU_ALWAYS_ASSERT(ret); auto blockInfo = readTopoBlockInfo(fh); @@ -668,11 +667,12 @@ void readMesh(const char* meshfilename, const char* fathers2Dfilename, const char* solutionfilename, const char* connHeadfilename, - MeshInfo& mesh) { + MeshInfo& mesh, + pcu::PCU *PCUObj) { mesh.dim = 3; //FIXME - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); char filename[1024]; snprintf(filename, 1024, "%s.%d",coordfilename,self); @@ -681,9 +681,9 @@ void readMesh(const char* meshfilename, PCU_ALWAYS_ASSERT(fc); getNumVerts(fc,mesh.numVerts); mesh.localNumVerts=mesh.numVerts; - mesh.numVerts=PCU_Add_Long(mesh.numVerts); + mesh.numVerts=PCUObj->Add(mesh.numVerts); - if(!PCU_Comm_Self()) + if(!PCUObj->Self()) fprintf(stderr, "numVerts %ld\n", mesh.numVerts); readCoords(fc, mesh.localNumVerts, &(mesh.coords)); fclose(fc); @@ -715,7 +715,7 @@ void readMesh(const char* meshfilename, snprintf(filename, 1024, "%s.%d",matchfilename,self); FILE* fm = fopen(filename, "r"); PCU_ALWAYS_ASSERT(fm); - readMatches(fm, mesh.numVerts, mesh.localNumVerts, &(mesh.matches)); + readMatches(fm, mesh.numVerts, mesh.localNumVerts, &(mesh.matches), PCUObj); fclose(fm); } @@ -725,7 +725,7 @@ void readMesh(const char* meshfilename, PCU_ALWAYS_ASSERT(meshConnStream.is_open()); std::ifstream connHeadStream(connHeadfilename, std::ios::in); PCU_ALWAYS_ASSERT(connHeadStream.is_open()); - auto blockInfo = readHeader(connHeadStream); + auto blockInfo = readHeader(connHeadStream, PCUObj); connHeadStream.close(); bool rewind = true; for(auto b : blockInfo) { @@ -744,11 +744,12 @@ void readMesh(const char* meshfilename, int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); int noVerify=0; // maintain default of verifying if not explicitly requesting it off if( argc < 11 ) { - if( !PCU_Comm_Self() ) { + if( !PCUObj.Self() ) { printf("Usage: %s " " " " " @@ -769,19 +770,19 @@ int main(int argc, char** argv) if( argc == 11 ) noVerify=atoi(argv[10]); - double t0 = PCU_Time(); + double t0 = pcu::Time(); MeshInfo m; - readMesh(argv[2],argv[3],argv[4],argv[5],argv[6],argv[7],argv[8],m); + readMesh(argv[2],argv[3],argv[4],argv[5],argv[6],argv[7],argv[8],m,&PCUObj); bool isMatched = true; if( !strcmp(argv[3], "NULL") ) isMatched = false; - if(!PCU_Comm_Self()) + if(!PCUObj.Self()) fprintf(stderr, "isMatched %d\n", isMatched); gmi_model* model = gmi_load(argv[1]); - apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched); + apf::Mesh2* mesh = apf::makeEmptyMdsMesh(model, m.dim, isMatched, &PCUObj); apf::GlobalToVert outMap; for( size_t i=0; i< m.elements.size(); i++) { apf::assemble(mesh, m.elements[i], m.numElms[i], m.elementType[i], outMap); @@ -807,7 +808,7 @@ int main(int argc, char** argv) if( strcmp(argv[6], "NULL") ) { setMappedTag(mesh, "fathers2D", m.fathers2D, 1, m.localNumVerts, outMap); delete [] m.fathers2D; - } else if(!PCU_Comm_Self()) + } else if(!mesh->getPCU()->Self()) fprintf(stderr, "fathers2D not requested \n"); if(0==1) { @@ -816,8 +817,8 @@ int main(int argc, char** argv) (void) ts; } - if(!PCU_Comm_Self()) - fprintf(stderr, "seconds to create mesh %.3f\n", PCU_Time()-t0); + if(!mesh->getPCU()->Self()) + fprintf(stderr, "seconds to create mesh %.3f\n", pcu::Time()-t0); if(noVerify != 1) mesh->verify(); outMap.clear(); @@ -827,6 +828,6 @@ int main(int argc, char** argv) mesh->destroyNative(); apf::destroyMesh(mesh); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/measureAnisoStats.cc b/test/measureAnisoStats.cc index 4bedbe673..c748303b2 100644 --- a/test/measureAnisoStats.cc +++ b/test/measureAnisoStats.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include @@ -37,16 +36,17 @@ void writeTable(const char* outfile, void getStats( const char* modelFile, const char* meshFile, const char* sizeName, const char* frameName, - const char* outputPrefix); + const char* outputPrefix, pcu::PCU *PCUObj); int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc < 5) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE1: %s " "\n", argv[0]); printf("USAGE2: %s " @@ -77,7 +77,7 @@ int main(int argc, char** argv) safe_mkdir(inPrefix); - getStats(".null", meshFile, sizeName, frameName, inPrefix); + getStats(".null", meshFile, sizeName, frameName, inPrefix, &PCUObj); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); @@ -88,8 +88,7 @@ int main(int argc, char** argv) SimModel_stop(); Sim_unregisterAllKeys(); #endif - - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -125,7 +124,7 @@ void writeTable(const char* outFile, void getStats( const char* modelFile, const char* meshFile, const char* sizeName, const char* frameName, - const char* outputPrefix) + const char* outputPrefix, pcu::PCU *PCUObj) { apf::Mesh2* m; @@ -133,12 +132,12 @@ void getStats( /* if it is a simmetrix mesh */ if (ph::mesh_has_ext(meshFile, "sms")) { pParMesh sim_mesh = PM_load(meshFile, NULL, NULL); - m = apf::createMesh(sim_mesh); + m = apf::createMesh(sim_mesh, PCUObj); } else #endif { // load the mesh change to desired order and write as before vtks - m = apf::loadMdsMesh(modelFile,meshFile); + m = apf::loadMdsMesh(modelFile,meshFile, PCUObj); } m->verify(); @@ -182,7 +181,7 @@ void getStats( m->verify(); - if (PCU_Comm_Self() == 0) + if (m->getPCU()->Self() == 0) printf("\n evaluating the statistics! \n"); // get the stats ma::SizeField* sf = ma::makeSizeField(m, sizes, frames, true); @@ -235,8 +234,8 @@ void getStats( const char* pathName = tmp.c_str(); safe_mkdir(pathName); - ssq << pathName << "/linearQTable_" << PCU_Comm_Self() << ".dat"; - sse << pathName << "/linearETable_" << PCU_Comm_Self() << ".dat"; + ssq << pathName << "/linearQTable_" << m->getPCU()->Self() << ".dat"; + sse << pathName << "/linearETable_" << m->getPCU()->Self() << ".dat"; ssm << pathName << "/mesh_quality_vis"; ssl << pathName << "/mesh_edge_length_vis"; @@ -300,7 +299,7 @@ void getStats( } std::stringstream sstq; - sstq << pathName << "/linearBLTriQTable_" << PCU_Comm_Self() << ".dat"; + sstq << pathName << "/linearBLTriQTable_" << m->getPCU()->Self() << ".dat"; writeTable(sstq.str().c_str(), tri_qtable); } #endif diff --git a/test/measureIsoStats.cc b/test/measureIsoStats.cc index ed9dd09e7..4ebacec5e 100644 --- a/test/measureIsoStats.cc +++ b/test/measureIsoStats.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include @@ -22,6 +21,7 @@ #include #include #include +#include // === includes for safe_mkdir === #include @@ -37,16 +37,17 @@ void writeTable(const char* outfile, void getStats( const char* modelFile, const char* meshFile, const char* sizeName, - const char* outputPrefix); + const char* outputPrefix, pcu::PCU *PCUObj); int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc < 4) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE1: %s " "\n", argv[0]); printf("USAGE2: %s " @@ -76,7 +77,7 @@ int main(int argc, char** argv) safe_mkdir(inPrefix); - getStats(".null", meshFile, sizeName, inPrefix); + getStats(".null", meshFile, sizeName, inPrefix, &PCUObj); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); @@ -87,8 +88,7 @@ int main(int argc, char** argv) SimModel_stop(); Sim_unregisterAllKeys(); #endif - - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -124,7 +124,7 @@ void writeTable(const char* outFile, void getStats( const char* modelFile, const char* meshFile, const char* sizeName, - const char* outputPrefix) + const char* outputPrefix, pcu::PCU *PCUObj) { apf::Mesh2* m; @@ -132,12 +132,12 @@ void getStats( /* if it is a simmetrix mesh */ if (ph::mesh_has_ext(meshFile, "sms")) { pParMesh sim_mesh = PM_load(meshFile, NULL, NULL); - m = apf::createMesh(sim_mesh); + m = apf::createMesh(sim_mesh, PCUObj); } else #endif { // load the mesh change to desired order and write as before vtks - m = apf::loadMdsMesh(modelFile,meshFile); + m = apf::loadMdsMesh(modelFile,meshFile,PCUObj); } m->verify(); @@ -173,7 +173,7 @@ void getStats( m->verify(); - if (PCU_Comm_Self() == 0) + if (m->getPCU()->Self() == 0) printf("\n evaluating the statistics! \n"); // get the stats ma::SizeField* sf = ma::makeSizeField(m, sizes); @@ -226,8 +226,8 @@ void getStats( const char* pathName = tmp.c_str(); safe_mkdir(pathName); - ssq << pathName << "/linearQTable_" << PCU_Comm_Self() << ".dat"; - sse << pathName << "/linearETable_" << PCU_Comm_Self() << ".dat"; + ssq << pathName << "/linearQTable_" << m->getPCU()->Self() << ".dat"; + sse << pathName << "/linearETable_" << m->getPCU()->Self() << ".dat"; ssm << pathName << "/mesh_quality_vis"; ssl << pathName << "/mesh_edge_length_vis"; @@ -291,7 +291,7 @@ void getStats( } std::stringstream sstq; - sstq << pathName << "/linearBLTriQTable_" << PCU_Comm_Self() << ".dat"; + sstq << pathName << "/linearBLTriQTable_" << m->getPCU()->Self() << ".dat"; writeTable(sstq.str().c_str(), tri_qtable); } #endif diff --git a/test/mixedNumbering.cc b/test/mixedNumbering.cc index 7c3fe5533..9017b454d 100644 --- a/test/mixedNumbering.cc +++ b/test/mixedNumbering.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -25,14 +24,14 @@ static void test_numbering(apf::Mesh* m) { std::vector global; int num_owned = apf::numberOwned(fields, owned); int num_ghost = apf::numberGhost(fields, ghost); - apf::makeGlobal(owned, global); + apf::makeGlobal(owned, global, m->getPCU()); for (size_t n=0; n < owned.size(); ++n) apf::destroyNumbering(owned[n]); for (size_t n=0; n < global.size(); ++n) apf::synchronize(global[n]); - PCU_Debug_Open(); - PCU_Debug_Print("number owned: %d\n", num_owned); - PCU_Debug_Print("number ghost: %d\n", num_ghost); + m->getPCU()->DebugOpen(); + m->getPCU()->DebugPrint("number owned: %d\n", num_owned); + m->getPCU()->DebugPrint("number ghost: %d\n", num_ghost); } static void write_output(apf::Mesh* m, const char* out) { @@ -49,15 +48,16 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); apf::reorderMdsMesh(m); test_numbering(m); write_output(m, argv[3]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/mkmodel.cc b/test/mkmodel.cc index 3c0da2377..3d5e063f3 100644 --- a/test/mkmodel.cc +++ b/test/mkmodel.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -11,22 +10,23 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 3 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.Self() ) printf("Create a discrete geometric model from a mesh\n" "Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); - apf::Mesh2* m = apf::loadMdsMesh(".null", argv[1]); + apf::Mesh2* m = apf::loadMdsMesh(".null", argv[1], &pcu_obj); gmi_model* g = m->getModel(); gmi_write_dmg(g, argv[2]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/mktopomodel.cc b/test/mktopomodel.cc index af97e9f4c..e0b64d360 100644 --- a/test/mktopomodel.cc +++ b/test/mktopomodel.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -11,10 +10,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 3 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.Self() ) printf("Create a topological geometric model from a mesh\n" "Usage: %s \n", argv[0]); MPI_Finalize(); @@ -22,13 +22,13 @@ int main(int argc, char** argv) } gmi_register_null(); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(".null", argv[1]); + apf::Mesh2* m = apf::loadMdsMesh(".null", argv[1], &pcu_obj); apf::deriveMdsModel(m); gmi_model* g = m->getModel(); gmi_write_dmg(g, argv[2]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/modelInfo.cc b/test/modelInfo.cc index 672a1474e..f35450f7d 100644 --- a/test/modelInfo.cc +++ b/test/modelInfo.cc @@ -1,7 +1,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -10,14 +9,17 @@ #include #endif #include //exit and exit_failure +#include +#include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 2 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -75,7 +77,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/moving.cc b/test/moving.cc index 1ee858796..7e297fe32 100644 --- a/test/moving.cc +++ b/test/moving.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include #include @@ -20,14 +19,15 @@ static void writeStep(apf::Mesh* m, int i) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 3 ) { fprintf(stderr, "Usage: %s \n", argv[0]); return 0; } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); dsp::Boundary moving; moving.insert(m->findModelEntity(2, 57)); moving.insert(m->findModelEntity(2, 62)); @@ -66,7 +66,7 @@ int main(int argc, char** argv) delete adapter; m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/nedelecShapes.cc b/test/nedelecShapes.cc index 9575e948e..534f4d04b 100644 --- a/test/nedelecShapes.cc +++ b/test/nedelecShapes.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -28,12 +27,13 @@ void testNedelec( int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(0); if (argc != 3) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -43,7 +43,7 @@ int main(int argc, char** argv) gmi_register_null(); gmi_model* g = gmi_load(argv[1]); - apf::Mesh2* m = apf::loadMdsMesh(g,argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(g,argv[2],&PCUObj); m->verify(); @@ -66,7 +66,7 @@ int main(int argc, char** argv) } apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/nektar_align.cc b/test/nektar_align.cc index 68b633203..62010da87 100644 --- a/test/nektar_align.cc +++ b/test/nektar_align.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -110,7 +109,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -120,7 +120,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); alignForNektar(m); m->writeNative(argv[3]); m->destroyNative(); @@ -131,6 +131,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/neper.cc b/test/neper.cc index 526ddf216..8be481036 100644 --- a/test/neper.cc +++ b/test/neper.cc @@ -2,28 +2,28 @@ #include #include #include -#include #include #include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsFromGmsh(gmi_load(argv[1]), argv[2]); + apf::Mesh2* m = apf::loadMdsFromGmsh(gmi_load(argv[1]), argv[2], &pcu_obj); m->verify(); m->writeNative(argv[3]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/newdim.cc b/test/newdim.cc index c1a7afece..29b68d160 100644 --- a/test/newdim.cc +++ b/test/newdim.cc @@ -2,17 +2,17 @@ #include #include #include -#include #include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_null(); gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 2, false, &PCUObj); apf::Vector3 points[4] = { apf::Vector3(0,0,0), apf::Vector3(1,0,0), @@ -39,7 +39,7 @@ int main(int argc, char** argv) m->verify(); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/osh2smb.cc b/test/osh2smb.cc index ed868ffe9..c45eb6777 100644 --- a/test/osh2smb.cc +++ b/test/osh2smb.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -16,15 +15,15 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc != 4) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.Self() == 0) { std::cout << "\n"; std::cout << "usage: osh2smb in.osh in.dmg out.smb\n"; std::cout << " or: osh2smb (usage)\n"; } - PCU_Comm_Free(); MPI_Finalize(); exit(EXIT_FAILURE); } @@ -35,12 +34,12 @@ int main(int argc, char** argv) { auto lib = Omega_h::Library(&argc, &argv); Omega_h::Mesh om(&lib); Omega_h::binary::read(argv[1], lib.world(), &om); - apf::Mesh2* am = apf::makeEmptyMdsMesh(model, om.dim(), false); + apf::Mesh2* am = apf::makeEmptyMdsMesh(model, om.dim(), false, &PCUObj); apf::from_omega_h(am, &om); am->writeNative(argv[3]); am->destroyNative(); apf::destroyMesh(am); } - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ph_adapt.cc b/test/ph_adapt.cc index 927ca1aca..72a9a7d20 100644 --- a/test/ph_adapt.cc +++ b/test/ph_adapt.cc @@ -37,8 +37,8 @@ static bool overwriteAPFCoord(apf::Mesh2* m) { return true; } -static FILE* openfile_read(ph::Input&, const char* path) { - return pcu_group_open(path, false); +static FILE* openfile_read(ph::Input&, const char* path, pcu::PCU *PCUObj) { + return pcu_group_open(PCUObj->GetCHandle(), path, false); } int main(int argc, char** argv) @@ -47,7 +47,8 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -58,10 +59,10 @@ int main(int argc, char** argv) #endif gmi_register_mesh(); /* load model, mesh and configure input */ - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); m->verify(); ph::Input in; - in.load("adapt.inp"); + in.load("adapt.inp", &PCUObj); in.openfile_read = openfile_read; /* attach solution and other fields */ ph::readAndAttachFields(in,m); @@ -88,7 +89,7 @@ int main(int argc, char** argv) ma::adapt(ma_in); m->verify(); apf::writeVtkFiles("after",m); - if (in.prePhastaBalanceMethod != "none" && PCU_Comm_Peers() > 1) + if (in.prePhastaBalanceMethod != "none" && m->getPCU()->Peers() > 1) ph::balance(in,m); /* output restart and geombc */ chef::preprocess(m,in); @@ -100,7 +101,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/poisson.cc b/test/poisson.cc index 62008c5d2..1d97d048c 100644 --- a/test/poisson.cc +++ b/test/poisson.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -57,8 +56,8 @@ class Poisson { apf::destroyMesh(mesh); } - void run() { - setup_grid(); + void run(pcu::PCU *PCUObj) { + setup_grid(PCUObj); setup_lin_alg(); fill_volumetric(); fill_boundary(); @@ -95,14 +94,14 @@ class Poisson { apf::NewArray GBF; apf::NewArray numbers; - void setup_grid() { + void setup_grid(pcu::PCU *PCUObj) { int nx = num_grid; int ny = (num_dims > 1) ? num_grid : 0; int nz = (num_dims > 2) ? num_grid : 0; double wx = 1.0; double wy = (num_dims > 1) ? 1.0 : 0.0; double wz = (num_dims > 2) ? 1.0 : 0.0; - mesh = apf::makeMdsBox(nx, ny, nz, wx, wy, wz, true); + mesh = apf::makeMdsBox(nx, ny, nz, wx, wy, wz, true, PCUObj); apf::reorderMdsMesh(mesh); shape = apf::getLagrange(p_order); sol = apf::createField(mesh, "u", apf::SCALAR, shape); @@ -237,12 +236,12 @@ class Poisson { }; -void test(int dim, int p) { +void test(int dim, int p, pcu::PCU *PCUObj) { int steps = 6-dim; int n_grid = 6-dim; for (int i=0; i < steps; ++i) { Poisson poisson(dim, p, n_grid); - poisson.run(); + poisson.run(PCUObj); n_grid *= 2; } } @@ -252,12 +251,13 @@ void test(int dim, int p) { int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 3); MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - PCU_ALWAYS_ASSERT(! PCU_Comm_Self()); + PCU_ALWAYS_ASSERT(! pcu_obj.Self()); int dim = atoi(argv[1]); int p = atoi(argv[2]); - test(dim, p); - PCU_Comm_Free(); + test(dim, p, &pcu_obj); + } MPI_Finalize(); } diff --git a/test/print_pumipic_partition.cc b/test/print_pumipic_partition.cc index 999156d6f..371a43614 100644 --- a/test/print_pumipic_partition.cc +++ b/test/print_pumipic_partition.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include @@ -20,16 +19,17 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 && argc != 6) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } - if (PCU_Comm_Peers() > 1) { - if ( !PCU_Comm_Self() ) + if (pcu_obj.Peers() > 1) { + if ( !pcu_obj.Self() ) printf("This tool must be run in serial.\n"); MPI_Finalize(); exit(EXIT_FAILURE); @@ -44,7 +44,7 @@ int main(int argc, char** argv) gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); int num_ranks = atoi(argv[3]); //Partition the mesh (Taken from zsplit.cc) @@ -80,6 +80,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ptnParma.cc b/test/ptnParma.cc index 765bf9bfc..d43a22a9f 100644 --- a/test/ptnParma.cc +++ b/test/ptnParma.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -85,22 +84,14 @@ apf::Migration* getPlan(apf::Mesh* m) return plan; } -void switchToMasters() +pcu::PCU* getGroupedPCU(pcu::PCU *PCUObj) { - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); int groupRank = self / partitionFactor; int group = self % partitionFactor; MPI_Comm groupComm; MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); - PCU_Switch_Comm(groupComm); -} - -void switchToAll() -{ - MPI_Comm prevComm = PCU_Get_Comm(); - PCU_Switch_Comm(MPI_COMM_WORLD); - MPI_Comm_free(&prevComm); - PCU_Barrier(); + return new pcu::PCU(groupComm); } void runParma(apf::Mesh2* m) { @@ -124,28 +115,31 @@ void runParma(apf::Mesh2* m) { Parma_PrintPtnStats(m, "final"); } -void mymain(bool ismaster) +void mymain(bool ismaster, pcu::PCU *PCUObj) { gmi_model* g = gmi_load(modelFile); apf::Mesh2* m = NULL; apf::Migration* plan = NULL; - switchToMasters(); + pcu::PCU *groupedPCUObj = getGroupedPCU(PCUObj); if (ismaster) { - m = apf::loadMdsMesh(modelFile,meshFile); + m = apf::loadMdsMesh(modelFile,meshFile,groupedPCUObj); Parma_PrintPtnStats(m, "initial"); plan = getPlan(m); } - switchToAll(); - m = apf::repeatMdsMesh(m, g, plan, partitionFactor); + //used switchPCU here to load the mesh on the groupedPCU, perform tasks and then call repeatMdsMesh + //on the globalPCU + if(m != nullptr) m->switchPCU(PCUObj); + delete groupedPCUObj; + m = apf::repeatMdsMesh(m, g, plan, partitionFactor, PCUObj); runParma(m); m->writeNative(outFile); freeMesh(m); } -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 8 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s " " <0:global|1:local>\n", argv[0]); MPI_Finalize(); @@ -158,7 +152,7 @@ void getConfig(int argc, char** argv) method = argv[5]; approach = argv[6]; isLocal = atoi(argv[7]); - if(!PCU_Comm_Self()) + if(!PCUObj->Self()) lion_eprint(1, "INPUTS model %s mesh %s out %s factor %d " "method %s approach %s isLocal %d\n", modelFile, meshFile, outFile, partitionFactor, method, approach, isLocal); @@ -169,9 +163,10 @@ void getConfig(int argc, char** argv) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - if( !PCU_Comm_Self() ) + if( !PCUObj.Self() ) lion_oprint(1, "PUMI version %s Git hash %s\n", pumi_version(), pumi_git_sha()); #ifdef HAVE_SIMMETRIX MS_init(); @@ -182,17 +177,17 @@ int main(int argc, char** argv) #endif gmi_register_mesh(); lion_set_verbosity(1); - getConfig(argc,argv); - if (PCU_Comm_Self() % partitionFactor) - mymain(false); + getConfig(argc,argv,&PCUObj); + if (PCUObj.Self() % partitionFactor) + mymain(false, &PCUObj); else - mymain(true); + mymain(true, &PCUObj); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); Sim_unregisterAllKeys(); SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/pumi.cc b/test/pumi.cc index bda9ed4a8..59aea11fd 100644 --- a/test/pumi.cc +++ b/test/pumi.cc @@ -37,10 +37,10 @@ struct testOwnership : public Ownership pOwnership o; }; -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc < 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -87,24 +87,26 @@ int main(int argc, char** argv) //********************************************************* { MPI_Init(&argc,&argv); - pumi_start(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pumi_load_pcu(&PCUObj); lion_set_verbosity(1); pumi_printSys(); #if 0 int i, processid = getpid(); - if (!PCU_Comm_Self()) + if (!PCUObj.Self()) { - std::cout<<"Proc "<> pid "<> pid "<>i; } else - std::cout<<"Proc "<> pid "<> pid "<begin(dim); while ((e = m->iterate(it))) { - pid=pumi_ment_getID(e)%PCU_Comm_Peers(); + pid=pumi_ment_getID(e)%m->getPCU()->Peers(); plan->send(e, pid); if (pid-1>=0) plan->send(e, pid-1); - if (pid+1send(e, pid+1); + if (pid+1getPCU()->Peers()) plan->send(e, pid+1); if (count==5) break; ++count; } @@ -249,7 +251,7 @@ int main(int argc, char** argv) // print elapsed time and increased heap memory pumi_printTimeMem("\n* [test_pumi] elapsed time and increased heap memory:", pumi_getTime()-begin_time, pumi_getMem()-begin_mem); - pumi_finalize(); + } MPI_Finalize(); } @@ -789,7 +791,7 @@ void TEST_GHOSTING(pMesh m) pumi_ghost_create(m, ghosting_plan); int total_mcount_diff=0, mcount_diff = pumi_mesh_getNumEnt(m, mesh_dim)-before_mcount; - MPI_Allreduce(&mcount_diff, &total_mcount_diff,1, MPI_INT, MPI_SUM, PCU_Get_Comm()); + MPI_Allreduce(&mcount_diff, &total_mcount_diff,1, MPI_INT, MPI_SUM, m->getPCU()->GetMPIComm()); if (!pumi_rank()) std::cout<<"\n[test_pumi] element-wise pumi_ghost_create: #ghost increase="<getPCU()->GetMPIComm()); if (!pumi_rank()) std::cout<<"\n[test_pumi] layer-wise pumi_ghost_createLayer (bd "<getPCU()->GetMPIComm()); if (!pumi_rank()) std::cout<<"\n[test_pumi] accumulative pumi_ghost_createLayer (bd "< #include #include #include @@ -19,18 +18,18 @@ double avgQuality = 0.0; double avgQualityBelowTol = 0.0; long numElemsBelowTol = 0; -static void fail(char** argv) +static void fail(char** argv, pcu::PCU *PCUObj) { - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } -static void getConfig(int argc, char** argv) +static void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if (argc != 4) - fail(argv); + fail(argv, PCUObj); modelFile = argv[1]; meshFile = argv[2]; qualityTol = atof(argv[3]); @@ -39,14 +38,14 @@ static void getConfig(int argc, char** argv) static void parallelReduce(apf::Mesh2* m) { long numElems = m->count(m->getDimension()); - numElems = PCU_Add_Long(numElems); - avgQuality = PCU_Add_Double(avgQuality); + numElems = m->getPCU()->Add(numElems); + avgQuality = m->getPCU()->Add(avgQuality); avgQuality /= numElems; - numElemsBelowTol = PCU_Add_Long(numElemsBelowTol); - avgQualityBelowTol = PCU_Add_Double(avgQualityBelowTol); + numElemsBelowTol = m->getPCU()->Add(numElemsBelowTol); + avgQualityBelowTol = m->getPCU()->Add(avgQualityBelowTol); if (numElemsBelowTol > 0) avgQualityBelowTol /= numElemsBelowTol; - minQuality = PCU_Min_Double(minQuality); + minQuality = m->getPCU()->Min(minQuality); } static void processMesh(apf::Mesh2* m) @@ -71,9 +70,9 @@ static void processMesh(apf::Mesh2* m) m->end(elems); } -void printDiagnostics() +void printDiagnostics(pcu::PCU *PCUObj) { - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) { printf("average element quality: %f\n", avgQuality); printf("quality tolerance: %f\n", qualityTol); @@ -91,15 +90,16 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + getConfig(argc,argv,&PCUObj); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); processMesh(m); - printDiagnostics(); + printDiagnostics(&PCUObj); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/refine2x.cc b/test/refine2x.cc index d09c090d5..c8e80b14d 100644 --- a/test/refine2x.cc +++ b/test/refine2x.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -85,10 +84,11 @@ class AnisotropicX: public ma::AnisotropicFunction { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc != 5) { - if(0==PCU_Comm_Self()) + if(0==PCUObj.Self()) std::cerr << "usage: " << argv[0] << " \n"; return EXIT_FAILURE; @@ -101,7 +101,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2]); + ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); AnisotropicX* ansx = new AnisotropicX(m, atoi(argv[3])); ma::Input* in = ma::makeAdvanced(ma::configure(m, ansx)); #ifdef PUMI_HAS_ZOLTAN @@ -129,6 +129,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/render.cc b/test/render.cc index 260186168..52ef4e3ab 100644 --- a/test/render.cc +++ b/test/render.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -18,10 +17,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -40,7 +40,7 @@ int main(int argc, char** argv) // does not fail when the input mesh is curved! crv::getBezier(2); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); std::string name = m->getShape()->getName(); int order = m->getShape()->getOrder(); @@ -59,7 +59,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/renderClass.cc b/test/renderClass.cc index 70441470b..2d110c685 100644 --- a/test/renderClass.cc +++ b/test/renderClass.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -34,10 +33,11 @@ static void number_dim(apf::Mesh* m, apf::FieldShape* shape, int dim, std::strin int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (!(argc == 4 || argc == 5)) { - if ( !PCU_Comm_Self() ) { + if ( !PCUObj.Self() ) { printf("Usage: %s \n", argv[0]); printf(" %s \n", argv[0]); } @@ -54,7 +54,7 @@ int main(int argc, char** argv) gmi_register_mesh(); char const* modelpath = argv[1]; char const* meshpath = argv[2]; - apf::Mesh2* m = apf::loadMdsMesh(modelpath, meshpath); + apf::Mesh2* m = apf::loadMdsMesh(modelpath, meshpath, &PCUObj); int dim; char const* vtkpath; if (argc == 5) { @@ -75,7 +75,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/render_ascii.cc b/test/render_ascii.cc index 7682aaf69..2d748f1ee 100644 --- a/test/render_ascii.cc +++ b/test/render_ascii.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -15,10 +14,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -31,7 +31,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); apf::writeASCIIVtkFiles(argv[3], m); m->destroyNative(); apf::destroyMesh(m); @@ -41,7 +41,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/reorder.cc b/test/reorder.cc index c95c888af..94d6670bb 100644 --- a/test/reorder.cc +++ b/test/reorder.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -16,10 +15,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -33,7 +33,7 @@ int main(int argc, char** argv) { #endif gmi_register_null(); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); apf::MeshTag* order = Parma_BfsReorder(m); apf::reorderMdsMesh(m, order); m->writeNative(argv[3]); @@ -45,6 +45,6 @@ int main(int argc, char** argv) { SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/repartition.cc b/test/repartition.cc index 4325116f0..07b85e6fd 100644 --- a/test/repartition.cc +++ b/test/repartition.cc @@ -17,16 +17,22 @@ const char* meshFile = 0; const char* outFile = 0; int inputPartCount = 1; +struct CreateGroupCommResult{ + bool isOriginal; + pcu::PCU *group_pcu_obj; +}; + void freeMesh(apf::Mesh* m) { m->destroyNative(); apf::destroyMesh(m); } -bool switchToOriginals() +CreateGroupCommResult createGroupComm(pcu::PCU *PCUObj) { - apf::Contract contract(inputPartCount, PCU_Comm_Peers()); - int self = PCU_Comm_Self(); + apf::Contract contract(inputPartCount, PCUObj->Peers()); + int self = PCUObj->Self(); + PCU_ALWAYS_ASSERT(self == PCUObj->Self()); int group; int groupRank; bool isOriginal = contract.isValid(self); @@ -41,22 +47,17 @@ bool switchToOriginals() } MPI_Comm groupComm; MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); - PCU_Switch_Comm(groupComm); - return isOriginal; + CreateGroupCommResult result; + result.isOriginal = isOriginal; + result.group_pcu_obj = new pcu::PCU(groupComm); + return result; } -void switchToAll() -{ - MPI_Comm prevComm = PCU_Get_Comm(); - PCU_Switch_Comm(MPI_COMM_WORLD); - MPI_Comm_free(&prevComm); - PCU_Barrier(); -} -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: mpirun -n %s" " \n" "Increase the part count of inMesh from inPartCount to outPartCount.\n" @@ -70,7 +71,7 @@ void getConfig(int argc, char** argv) inputPartCount = atoi(argv[2]); meshFile = argv[3]; outFile = argv[4]; - PCU_ALWAYS_ASSERT(inputPartCount <= PCU_Comm_Peers()); + PCU_ALWAYS_ASSERT(inputPartCount <= PCUObj->Peers()); } void balance(apf::Mesh2* m) @@ -92,25 +93,28 @@ void balance(apf::Mesh2* m) } + int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU expanded_pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - getConfig(argc,argv); + getConfig(argc,argv,&expanded_pcu_obj); gmi_model* g = gmi_load(modelFile); apf::Mesh2* m = 0; - bool isOriginal = switchToOriginals(); - if (isOriginal) - m = apf::loadMdsMesh(g, meshFile); - switchToAll(); - m = apf::expandMdsMesh(m, g, inputPartCount); + CreateGroupCommResult result = createGroupComm(&expanded_pcu_obj); + + if (result.isOriginal) + m = apf::loadMdsMesh(g, meshFile, result.group_pcu_obj); + + m = apf::expandMdsMesh(m, g, inputPartCount, &expanded_pcu_obj); balance(m); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); freeMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/reposition.cc b/test/reposition.cc index 48268fb0f..c3d4b47ce 100644 --- a/test/reposition.cc +++ b/test/reposition.cc @@ -2,19 +2,19 @@ #include #include #include -#include #include #include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #if 0 gmi_register_null(); gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, &PCUObj); apf::Vector3 vx[4] = {apf::Vector3(0,0,0), apf::Vector3(1,0,0), @@ -27,7 +27,7 @@ int main(int argc, char** argv) apf::MeshEntity* v = tv[0]; #else gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], &PCUObj); apf::MeshIterator* it = m->begin(0); apf::MeshEntity* v; while ((v = m->iterate(it))) { @@ -43,6 +43,6 @@ int main(int argc, char** argv) apf::writeVtkFiles("before", m); ma::repositionVertex(m, v, 20, 1.0); apf::writeVtkFiles("after", m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/residualErrorEstimation_test.cc b/test/residualErrorEstimation_test.cc index 085966460..a34934e5a 100644 --- a/test/residualErrorEstimation_test.cc +++ b/test/residualErrorEstimation_test.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -27,7 +26,8 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -37,7 +37,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); m->verify(); kappa = freq * M_PI; @@ -96,7 +96,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/rm_extrusion.cc b/test/rm_extrusion.cc index dcb9f2e86..f415fd11f 100644 --- a/test/rm_extrusion.cc +++ b/test/rm_extrusion.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -36,7 +35,7 @@ void M_removeSurfaceExtrusionConstraints(pMesh, pPList); #endif -void getConfig(int argc, char** argv) { +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { opterr = 0; @@ -63,18 +62,18 @@ void getConfig(int argc, char** argv) { gmi_native_path = optarg; break; case '?': - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf ("warning: skipping unrecognized option\n"); break; default: - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf("Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } } if(argc-optind != 3) { - if (!PCU_Comm_Self()) + if (!PCUObj->Self()) printf("Usage %s %s", argv[0], usage); exit(EXIT_FAILURE); } @@ -83,7 +82,7 @@ void getConfig(int argc, char** argv) { sms_path = argv[i++]; smsNew_path = argv[i++]; - if (!PCU_Comm_Self()) { + if (!PCUObj->Self()) { printf ("enable_log %d\n", should_log); printf ("native-model \'%s\' model \'%s\' simmetrix mesh \'%s\' output mesh \'%s\'\n", gmi_native_path, gmi_path, sms_path, smsNew_path); @@ -93,7 +92,8 @@ void getConfig(int argc, char** argv) { int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); MS_init(); SimAdvMeshing_start(); @@ -101,7 +101,7 @@ int main(int argc, char** argv) Sim_readLicenseFile(NULL); SimPartitionedMesh_start(&argc,&argv); - getConfig(argc, argv); + getConfig(argc, argv, &PCUObj); if( should_log ) Sim_logOn("rm_extrusion.sim.log"); @@ -116,21 +116,21 @@ int main(int argc, char** argv) else mdl = gmi_load(gmi_path); pGModel simModel = gmi_export_sim(mdl); - if(!PCU_Comm_Self()) + if(!PCUObj.Self()) fprintf(stderr, "Read model\n"); - double t0 = PCU_Time(); + double t0 = pcu::Time(); pMesh sim_mesh = M_load(sms_path, simModel, progress); - if(!PCU_Comm_Self()) + if(!PCUObj.Self()) fprintf(stderr, "Read mesh\n"); M_removeSurfaceExtrusionConstraints(sim_mesh, NULL); - if(!PCU_Comm_Self()) + if(!PCUObj.Self()) fprintf(stderr, "Removed surface extrusion constraints\n"); M_write(sim_mesh, smsNew_path, 0, progress); - double t1 = PCU_Time(); + double t1 = pcu::Time(); - if(!PCU_Comm_Self()) + if(!PCUObj.Self()) fprintf(stderr, "read the mesh, removed the face extrusion attributes, and wrote the mesh %f seconds\n", t1-t0); M_release(sim_mesh); @@ -144,6 +144,6 @@ int main(int argc, char** argv) MS_exit(); if( should_log ) Sim_logOff(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/runSimxAnisoAdapt.cc b/test/runSimxAnisoAdapt.cc index ff42e640e..81896ad28 100644 --- a/test/runSimxAnisoAdapt.cc +++ b/test/runSimxAnisoAdapt.cc @@ -6,7 +6,6 @@ #include #include #include -#include #include #include @@ -72,18 +71,19 @@ void addFields(apf::Mesh2* m, const vector &adaptedFrames); apf::Mesh2* convertToPumi( pMesh simxMesh, int dim, - const char* sizeName, const char* frameName); + const char* sizeName, const char* frameName, pcu::PCU *PCUObj); int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); MS_init(); // Call before calling Sim_readLicenseFile Sim_readLicenseFile(0); SimDiscrete_start(0); // initialize GeomSim Discrete library if (argc < 7) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE: %s " " \n", argv[0]); } @@ -94,7 +94,7 @@ int main(int argc, char** argv) gmi_register_mesh(); gmi_register_null(); - PCU_ALWAYS_ASSERT_VERBOSE(PCU_Comm_Peers() == 1, + PCU_ALWAYS_ASSERT_VERBOSE(PCUObj.Peers() == 1, "This utility only works for serial meshes!"); const char* inputPumiModel = argv[1]; @@ -118,7 +118,7 @@ int main(int argc, char** argv) snprintf(outImprovedSimxMesh, 256, "%s_adapted_improved.sms", prefix); snprintf(outImprovedPumiMesh, 256, "%s_adapted_improved.smb", prefix); - apf::Mesh2* m = apf::loadMdsMesh(inputPumiModel, inputPumiMesh); + apf::Mesh2* m = apf::loadMdsMesh(inputPumiModel, inputPumiMesh, &PCUObj); char message[512]; // first find the sizes field @@ -193,7 +193,7 @@ int main(int argc, char** argv) printf("%s\n", outAdaptedSimxMesh); printf("%s\n", outAdaptedPumiMesh); M_write(simxMesh,outAdaptedSimxMesh, 0,0); // write out the initial mesh data - apf::Mesh2* m2 = convertToPumi(simxMesh, dim, sizeName, frameName); + apf::Mesh2* m2 = convertToPumi(simxMesh, dim, sizeName, frameName, &PCUObj); m2->writeNative(outAdaptedPumiMesh); printf("===DONE===\n"); @@ -207,7 +207,7 @@ int main(int argc, char** argv) printf("%s\n", outImprovedSimxMesh); printf("%s\n", outImprovedPumiMesh); M_write(simxMesh,outImprovedSimxMesh, 0,0); // write out the initial mesh data - apf::Mesh2* m3 = convertToPumi(simxMesh, dim, sizeName, frameName); + apf::Mesh2* m3 = convertToPumi(simxMesh, dim, sizeName, frameName, &PCUObj); m3->writeNative(outImprovedPumiMesh); printf("===DONE===\n"); @@ -228,7 +228,7 @@ int main(int argc, char** argv) SimDiscrete_stop(0); Sim_unregisterAllKeys(); MS_exit(); - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -436,21 +436,21 @@ pMSAdapt addSizesToSimxMesh( double runSimxAdapt(pMSAdapt adapter) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); MSA_adapt(adapter, 0); MSA_delete(adapter); - double t1 = PCU_Time(); + double t1 = pcu::Time(); return t1 - t0; } double runSimxMeshImprover(pMesh mesh, double minQuality) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); pVolumeMeshImprover vmi = VolumeMeshImprover_new(mesh); VolumeMeshImprover_setShapeMetric(vmi, ShapeMetricType_VolLenRatio, minQuality); VolumeMeshImprover_execute(vmi, 0); VolumeMeshImprover_delete(vmi); - double t1 = PCU_Time(); + double t1 = pcu::Time(); return t1 - t0; } @@ -585,7 +585,8 @@ void addFields(apf::Mesh2* m, apf::Mesh2* convertToPumi( pMesh simxMesh, int dim, const char* sizeName, - const char* frameName) + const char* frameName, + pcu::PCU *PCUObj) { double* adaptedCoords; apf::Gid* adaptedConns; @@ -596,7 +597,7 @@ apf::Mesh2* convertToPumi( adaptedNumVerts, adaptedNumElems, adaptedSizes, adaptedFrames); gmi_model* nullModel = gmi_load(".null"); - apf::Mesh2* m2 = apf::makeEmptyMdsMesh(nullModel, dim, false); + apf::Mesh2* m2 = apf::makeEmptyMdsMesh(nullModel, dim, false, PCUObj); apf::GlobalToVert outMap; apf::construct(m2, adaptedConns, adaptedNumElems, apf::Mesh::TET, outMap);; apf::alignMdsRemotes(m2); diff --git a/test/scale.cc b/test/scale.cc index 3816dc940..d321a066a 100644 --- a/test/scale.cc +++ b/test/scale.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -16,8 +15,8 @@ struct Scale { double s; }; -static void print_usage(char** argv) { - if (! PCU_Comm_Self()) +static void print_usage(char** argv, pcu::PCU *PCUObj) { + if (! PCUObj->Self()) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -55,10 +54,11 @@ static void scale_mesh(apf::Mesh2* m, Scale const& s) { int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - if (argc != 8) print_usage(argv); + if (argc != 8) print_usage(argv, &PCUObj); const char* gfile = argv[1]; const char* mfile = argv[2]; const char* ofile = argv[3]; @@ -67,13 +67,13 @@ int main(int argc, char** argv) { scale.y = atof(argv[5]); scale.z = atof(argv[6]); scale.s = atof(argv[7]); - apf::Mesh2* m = apf::loadMdsMesh(gfile, mfile); + apf::Mesh2* m = apf::loadMdsMesh(gfile, mfile, &PCUObj); m->verify(); scale_mesh(m, scale); m->verify(); m->writeNative(ofile); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/serialize.cc b/test/serialize.cc index f233770b4..74b7c5c47 100644 --- a/test/serialize.cc +++ b/test/serialize.cc @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -27,10 +26,11 @@ struct GroupCode : public Parma_GroupCode int main( int argc, char* argv[]) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -46,9 +46,9 @@ int main( int argc, char* argv[]) gmi_register_null(); crv::getBezier(2);//hack to make sure curved meshes can be serialized! GroupCode code; - code.mesh = apf::loadMdsMesh(argv[1], argv[2]); + code.mesh = apf::loadMdsMesh(argv[1], argv[2], &pcu_obj); code.meshFile = argv[3]; - apf::Unmodulo outMap(PCU_Comm_Self(), PCU_Comm_Peers()); + apf::Unmodulo outMap(code.mesh->getPCU()->Self(), code.mesh->getPCU()->Peers()); Parma_ShrinkPartition(code.mesh, atoi(argv[4]), code); code.mesh->destroyNative(); apf::destroyMesh(code.mesh); @@ -58,7 +58,7 @@ int main( int argc, char* argv[]) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/shapefun.cc b/test/shapefun.cc index 00aaa7036..7c02b1d2b 100644 --- a/test/shapefun.cc +++ b/test/shapefun.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -206,10 +205,10 @@ void testQuadrilateralNodeValues() { testNodeValues(shp2, nodes2, 9); } -void testVolume(int type, apf::Vector3 const* points, double volume) +void testVolume(int type, apf::Vector3 const* points, double volume, pcu::PCU *PCUObj) { gmi_model* model = gmi_load(".null"); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, 3, false, PCUObj); apf::MeshEntity* e = apf::buildOneElement( m, m->findModelEntity(3,0), type, points); double v = apf::measure(m,e); @@ -218,7 +217,7 @@ void testVolume(int type, apf::Vector3 const* points, double volume) apf::destroyMesh(m); } -void testPrismVolume() +void testPrismVolume(pcu::PCU *PCUObj) { apf::Vector3 points[6] = {apf::Vector3(0,0,0), @@ -227,10 +226,10 @@ void testPrismVolume() apf::Vector3(0,0,1), apf::Vector3(1,0,1), apf::Vector3(0,1,1)}; - testVolume(apf::Mesh::PRISM, points, 0.5); + testVolume(apf::Mesh::PRISM, points, 0.5, PCUObj); } -void testPyramidVolume() +void testPyramidVolume(pcu::PCU *PCUObj) { apf::Vector3 points[5] = {apf::Vector3(0,0,0), @@ -238,13 +237,14 @@ void testPyramidVolume() apf::Vector3(1,1,0), apf::Vector3(0,1,0), apf::Vector3(0,0,3)}; - testVolume(apf::Mesh::PYRAMID, points, 1); + testVolume(apf::Mesh::PYRAMID, points, 1, PCUObj); } int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_null(); testP1LineNodeValues(); @@ -259,8 +259,8 @@ int main(int argc, char** argv) testPrismNodeValues(); testPyramidNodeValues(); testQuadrilateralNodeValues(); - testPrismVolume(); - testPyramidVolume(); - PCU_Comm_Free(); + testPrismVolume(&PCUObj); + testPyramidVolume(&PCUObj); + } MPI_Finalize(); } diff --git a/test/shapefun2.cc b/test/shapefun2.cc index 121b07f58..5922c36e7 100644 --- a/test/shapefun2.cc +++ b/test/shapefun2.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include @@ -131,7 +130,7 @@ static void checkEntityShape(apf::Mesh* lm, apf::Mesh* m, /* For every shape function we test, for every type we test, * create two meshes */ -static void checkFieldShape(apf::FieldShape* fs) +static void checkFieldShape(apf::FieldShape* fs, pcu::PCU *PCUObj) { for (int type = 1; type < apf::Mesh::TYPES; ++type){ apf::EntityShape* es = fs->getEntityShape(type); @@ -141,8 +140,8 @@ static void checkFieldShape(apf::FieldShape* fs) gmi_model* lmodel = gmi_load(".null"); gmi_model* model = gmi_load(".null"); - apf::Mesh2* lm = apf::makeEmptyMdsMesh(lmodel, typeDim, false); - apf::Mesh2* m = apf::makeEmptyMdsMesh(model, typeDim, false); + apf::Mesh2* lm = apf::makeEmptyMdsMesh(lmodel, typeDim, false, PCUObj); + apf::Mesh2* m = apf::makeEmptyMdsMesh(model, typeDim, false, PCUObj); apf::MeshEntity* le = apf::buildOneElement( lm, lm->findModelEntity(typeDim,0), type, points[type]); @@ -171,7 +170,8 @@ static void checkFieldShape(apf::FieldShape* fs) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_null(); @@ -180,9 +180,9 @@ int main(int argc, char** argv) crv::getBezier(1),crv::getBezier(2)}; for (int i = 0; i < 4; ++i) - test::checkFieldShape(fs[i]); + test::checkFieldShape(fs[i], &PCUObj); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/simZBalance.cc b/test/simZBalance.cc index 9494e4342..910da6a34 100644 --- a/test/simZBalance.cc +++ b/test/simZBalance.cc @@ -1,4 +1,3 @@ -#include #include #include #include @@ -12,25 +11,6 @@ #include #include -static void initialize(int argc, char** argv) { - MPI_Init(&argc, &argv); - PCU_Comm_Init(); - lion_set_verbosity(1); - Sim_readLicenseFile(NULL); - gmi_sim_start(); - gmi_register_sim(); - SimPartitionedMesh_start(&argc, &argv); - MS_init(); -} - -static void finalize() { - MS_exit(); - SimPartitionedMesh_stop(); - gmi_sim_stop(); - Sim_unregisterAllKeys(); - PCU_Comm_Free(); - MPI_Finalize(); -} static void load_balance(apf::Mesh2* m) { Parma_PrintPtnStats(m, "initial"); @@ -44,18 +24,33 @@ static void load_balance(apf::Mesh2* m) { } int main(int argc, char** argv) { - initialize(argc, argv); + MPI_Init(&argc, &argv); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + lion_set_verbosity(1); + Sim_readLicenseFile(NULL); + gmi_sim_start(); + gmi_register_sim(); + SimPartitionedMesh_start(&argc, &argv); + MS_init(); + const char* smd_file = argv[1]; const char* sms_file = argv[2]; gmi_model* apf_model = gmi_sim_load(0, smd_file); pGModel sim_model = gmi_export_sim(apf_model); pParMesh sim_mesh = PM_load(sms_file, sim_model, NULL); - apf::Mesh2* apf_mesh = apf::createMesh(sim_mesh); + apf::Mesh2* apf_mesh = apf::createMesh(sim_mesh, &PCUObj); //apf_mesh->verify(); <- this calls Simmetrix's verify function apf::verify(apf_mesh); load_balance(apf_mesh); apf_mesh->destroyNative(); apf::destroyMesh(apf_mesh); gmi_destroy(apf_model); - finalize(); + + MS_exit(); + SimPartitionedMesh_stop(); + gmi_sim_stop(); + Sim_unregisterAllKeys(); + } + MPI_Finalize(); } diff --git a/test/sim_part.cc b/test/sim_part.cc index fbf9e0f14..83906b02f 100644 --- a/test/sim_part.cc +++ b/test/sim_part.cc @@ -22,7 +22,6 @@ // /* cheap hackish way to get SIM_PARASOLID and SIM_ACIS */ -#include #include #include "gmi_sim_config.h" #include @@ -40,6 +39,8 @@ #endif #endif +#include +#include #include #include #include @@ -66,9 +67,10 @@ void messageHandler(int type, const char *msg); int main(int argc, char **argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); - PCU_Protect(); + pcu::Protect(); // Initialize PartitionedMesh - this should be the first Simmetrix call // Also initializes MPI in parallel SimPartitionedMesh_start(&argc, &argv); @@ -106,7 +108,7 @@ int main(int argc, char **argv) outmeshFilename = tmp.c_str(); /* print message */ - if (PCU_Comm_Self()==0) { + if (PCUObj.Self()==0) { cout< #include #include -#include #include #include #include @@ -16,21 +15,21 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc != 4) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.Self() == 0) { std::cout << "\n"; std::cout << "usage: smb2osh in.dmg in.smb out.osh\n"; std::cout << " or: smb2osh (usage)\n"; } - PCU_Comm_Free(); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); gmi_register_null(); - apf::Mesh2* am = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* am = apf::loadMdsMesh(argv[1], argv[2], &PCUObj); { auto lib = Omega_h::Library(&argc, &argv); Omega_h::Mesh om(&lib); @@ -39,6 +38,6 @@ int main(int argc, char** argv) { apf::destroyMesh(am); Omega_h::binary::write(argv[3], &om); } - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/snap.cc b/test/snap.cc index 6134d245d..13953fa2b 100644 --- a/test/snap.cc +++ b/test/snap.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -13,14 +12,15 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); MS_init(); SimModel_start(); Sim_readLicenseFile(0); gmi_sim_start(); gmi_register_sim(); - ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2]); + ma::Mesh* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); const ma::Input* in = ma::configureIdentity(m); ma::adapt(in); m->writeNative(argv[3]); @@ -30,7 +30,7 @@ int main(int argc, char** argv) Sim_unregisterAllKeys(); SimModel_stop(); MS_exit(); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/split.cc b/test/split.cc index e56b5dc7c..6dbcc4c18 100644 --- a/test/split.cc +++ b/test/split.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -38,28 +37,20 @@ apf::Migration* getPlan(apf::Mesh* m) return plan; } -void switchToOriginals() +pcu::PCU* getGroupedPCU(pcu::PCU *PCUObj) { - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); int groupRank = self / partitionFactor; int group = self % partitionFactor; MPI_Comm groupComm; MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); - PCU_Switch_Comm(groupComm); + return new pcu::PCU(groupComm); } -void switchToAll() -{ - MPI_Comm prevComm = PCU_Get_Comm(); - PCU_Switch_Comm(MPI_COMM_WORLD); - MPI_Comm_free(&prevComm); - PCU_Barrier(); -} - -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -68,7 +59,7 @@ void getConfig(int argc, char** argv) meshFile = argv[2]; outFile = argv[3]; partitionFactor = atoi(argv[4]); - PCU_ALWAYS_ASSERT(partitionFactor <= PCU_Comm_Peers()); + PCU_ALWAYS_ASSERT(partitionFactor <= PCUObj->Peers()); } } @@ -76,7 +67,8 @@ void getConfig(int argc, char** argv) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -86,19 +78,22 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv); - bool isOriginal = ((PCU_Comm_Self() % partitionFactor) == 0); + getConfig(argc,argv,&PCUObj); + bool isOriginal = ((PCUObj.Self() % partitionFactor) == 0); gmi_model* g = 0; g = gmi_load(modelFile); apf::Mesh2* m = 0; apf::Migration* plan = 0; - switchToOriginals(); + pcu::PCU *groupedPCUObj = getGroupedPCU(&PCUObj); if (isOriginal) { - m = apf::loadMdsMesh(g, meshFile); + m = apf::loadMdsMesh(g, meshFile, groupedPCUObj); plan = getPlan(m); } - switchToAll(); - m = repeatMdsMesh(m, g, plan, partitionFactor); + //used switchPCU here to load the mesh on the groupedPCU, perform tasks and then call repeatMdsMesh + //on the globalPCU + if(m != nullptr) m->switchPCU(&PCUObj); + delete groupedPCUObj; + m = repeatMdsMesh(m, g, plan, partitionFactor, &PCUObj); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); freeMesh(m); @@ -108,7 +103,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/spr_test.cc b/test/spr_test.cc index 330d45e25..69276fdb2 100644 --- a/test/spr_test.cc +++ b/test/spr_test.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -19,10 +18,11 @@ int main(int argc, char** argv) const char* outFile = argv[3]; const int order = atoi(argv[4]); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* mesh = apf::loadMdsMesh(modelFile, meshFile); + apf::Mesh2* mesh = apf::loadMdsMesh(modelFile, meshFile, &PCUObj); if (mesh->findTag("coordinates_edg")) mesh->changeShape(apf::getSerendipity(), false); apf::Field* f = @@ -30,13 +30,13 @@ int main(int argc, char** argv) apf::Field* eps = spr::getGradIPField(f, "eps", order); apf::destroyField(f); double adaptRatio = 0.1; - apf::Field* sizef = spr::getSPRSizeField(eps,adaptRatio); + apf::Field* sizef = spr::getSPRSizeField(eps,adaptRatio,&PCUObj); apf::destroyField(eps); writeVtkFiles(outFile,mesh); apf::destroyField(sizef); mesh->destroyNative(); apf::destroyMesh(mesh); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/swapDoubles.cc b/test/swapDoubles.cc index daba9467c..4b9e8a9f1 100644 --- a/test/swapDoubles.cc +++ b/test/swapDoubles.cc @@ -6,7 +6,8 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); const size_t n = 2; double *d_orig = new double[n]; std::iota(d_orig,d_orig+n,0); @@ -25,7 +26,7 @@ int main(int argc, char** argv) { } delete [] d_orig; delete [] d; - PCU_Comm_Free(); + } MPI_Finalize(); return 0; } diff --git a/test/test_integrator.cc b/test/test_integrator.cc index 682ea2eee..41f1cd320 100644 --- a/test/test_integrator.cc +++ b/test/test_integrator.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -24,13 +23,14 @@ class CountIntegrator : public apf::Integrator { }; int main(int argc, char ** argv) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); // argument should be model, mesh PCU_ALWAYS_ASSERT(argc == 3); gmi_register_mesh(); gmi_register_null(); - apf::Mesh2* mesh = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* mesh = apf::loadMdsMesh(argv[1], argv[2], &PCUObj); CountIntegrator * countInt = new CountIntegrator(); // test integration over implicitly defined mesh dimension countInt->process(mesh); @@ -47,7 +47,7 @@ int main(int argc, char ** argv) { delete countInt; mesh->destroyNative(); apf::destroyMesh(mesh); - PCU_Comm_Free(); + } MPI_Finalize(); return 0; } diff --git a/test/test_matrix_grad.cc b/test/test_matrix_grad.cc index 71eb13729..80d702927 100644 --- a/test/test_matrix_grad.cc +++ b/test/test_matrix_grad.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -94,10 +93,11 @@ int main(int argc, char* argv[]) return 1; } MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* mesh = apf::loadMdsMesh(argv[1], argv[2]); + apf::Mesh2* mesh = apf::loadMdsMesh(argv[1], argv[2], &pcu_obj); int order=1; apf::Field* nodal_matrix_fld = apf::createLagrangeField(mesh, "matrix", apf::MATRIX, order); apf::Field* matrix_deriv = apf::createPackedField(mesh, "matrix_deriv", 27, apf::getIPShape(3,order)); @@ -116,7 +116,7 @@ int main(int argc, char* argv[]) std::cout<<"Done"<destroyNative(); apf::destroyMesh(mesh); - PCU_Comm_Free(); + } MPI_Finalize(); return 0; } diff --git a/test/test_scaling.cc b/test/test_scaling.cc index 0a18cddea..f727a902b 100644 --- a/test/test_scaling.cc +++ b/test/test_scaling.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -14,10 +13,11 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); apf::Field* identity_size = samSz::isoSize(m); double scaling_factor = sam::getIsoLengthScalar(identity_size, m->count(m->getDimension())); @@ -26,6 +26,6 @@ int main(int argc, char** argv) PCU_ALWAYS_ASSERT(0.5 < scaling_factor); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/test_verify.cc b/test/test_verify.cc index fdc57e0f1..9802b09ad 100644 --- a/test/test_verify.cc +++ b/test/test_verify.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -12,11 +11,12 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); PCU_ALWAYS_ASSERT(argc == 3); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); apf::DynamicArray fields(10); apf::FieldShape* shapes[10] = { apf::getLagrange(1), @@ -43,6 +43,6 @@ int main(int argc, char** argv) apf::destroyField(fields[i]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/tetrahedronize.cc b/test/tetrahedronize.cc index edd035d3f..ee0555833 100644 --- a/test/tetrahedronize.cc +++ b/test/tetrahedronize.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -16,7 +15,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -26,7 +26,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(argv[1], argv[2]); + ma::Mesh* m = apf::loadMdsMesh(argv[1], argv[2], &PCUObj); ma::Input* in = ma::makeAdvanced(ma::configureIdentity(m)); in->shouldTurnLayerToTets = true; ma::adapt(in); @@ -39,7 +39,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/torus_ma_test.cc b/test/torus_ma_test.cc index d7d6bcc06..5c0799848 100644 --- a/test/torus_ma_test.cc +++ b/test/torus_ma_test.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -48,10 +47,11 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); m->verify(); apf::writeVtkFiles("torus_before",m); CylindricalShock sf(m); @@ -69,7 +69,7 @@ int main(int argc, char** argv) apf::writeVtkFiles("torus_after",m); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ugrid.cc b/test/ugrid.cc index 87ef32016..ab35ff931 100644 --- a/test/ugrid.cc +++ b/test/ugrid.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -19,56 +18,53 @@ apf::Migration* getPlan(apf::Mesh* m, const int partitionFactor) return plan; } -void switchToOriginals(const int partitionFactor) +pcu::PCU* getGroupedPCU(const int partitionFactor, pcu::PCU *PCUObj) { - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); int groupRank = self / partitionFactor; int group = self % partitionFactor; MPI_Comm groupComm; MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); - PCU_Switch_Comm(groupComm); + return new pcu::PCU(groupComm); } -void switchToAll() -{ - MPI_Comm prevComm = PCU_Get_Comm(); - PCU_Switch_Comm(MPI_COMM_WORLD); - MPI_Comm_free(&prevComm); - PCU_Barrier(); -} int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_null(); const int partitionFactor = atoi(argv[4]); - PCU_ALWAYS_ASSERT(partitionFactor <= PCU_Comm_Peers()); - bool isOriginal = ((PCU_Comm_Self() % partitionFactor) == 0); + PCU_ALWAYS_ASSERT(partitionFactor <= PCUObj.Peers()); + bool isOriginal = ((PCUObj.Self() % partitionFactor) == 0); gmi_model* g = gmi_load(".null"); apf::Mesh2* m = 0; apf::Migration* plan = 0; - switchToOriginals(partitionFactor); + pcu::PCU *groupedPCUObj = getGroupedPCU(partitionFactor, &PCUObj); if (isOriginal) { - m = apf::loadMdsFromUgrid(g, argv[1]); + m = apf::loadMdsFromUgrid(g, argv[1], groupedPCUObj); apf::deriveMdsModel(m); m->verify(); plan = getPlan(m, partitionFactor); } - switchToAll(); - m = repeatMdsMesh(m, g, plan, partitionFactor); + //used switchPCU here to load the mesh on the groupedPCU, perform tasks and then call repeatMdsMesh + //on the globalPCU + if(m != nullptr) m->switchPCU(&PCUObj); + delete groupedPCUObj; + m = repeatMdsMesh(m, g, plan, partitionFactor, &PCUObj); Parma_PrintPtnStats(m, ""); gmi_write_dmg(g,argv[2]); m->writeNative(argv[3]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/ugridptnstats.cc b/test/ugridptnstats.cc index d974131d4..c9a67dac4 100644 --- a/test/ugridptnstats.cc +++ b/test/ugridptnstats.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -20,7 +19,8 @@ const double weights[8] = {vtxw, edgew, triw, quadw, tetw, hexw, przw, pyrw}; int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_null(); PCU_ALWAYS_ASSERT( 3 == argc ); @@ -28,7 +28,7 @@ int main(int argc, char** argv) const char* ptnfile = argv[2]; gmi_register_null(); gmi_model* g = gmi_load(".null"); - apf::printUgridPtnStats(g,ugridfile,ptnfile,weights); - PCU_Comm_Free(); + apf::printUgridPtnStats(g,ugridfile,ptnfile,weights,&PCUObj); + } MPI_Finalize(); } diff --git a/test/uniform.cc b/test/uniform.cc index 427598a06..2d45b167b 100644 --- a/test/uniform.cc +++ b/test/uniform.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -17,10 +16,10 @@ const char* modelFile = 0; const char* meshFile = 0; const char* outFile = 0; -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -34,7 +33,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -44,8 +44,8 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + getConfig(argc,argv,&PCUObj); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); ma::Input* in = ma::makeAdvanced(ma::configureUniformRefine(m, 1)); if (in->shouldSnap) { in->shouldSnap = false; @@ -62,7 +62,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/verify.cc b/test/verify.cc index 4e2b39ffe..76f94e055 100644 --- a/test/verify.cc +++ b/test/verify.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -16,7 +15,8 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -26,7 +26,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); m->verify(); m->destroyNative(); apf::destroyMesh(m); @@ -36,7 +36,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/verify_2nd_order_shapes.cc b/test/verify_2nd_order_shapes.cc index bc26005eb..acfc64f48 100644 --- a/test/verify_2nd_order_shapes.cc +++ b/test/verify_2nd_order_shapes.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -18,10 +17,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 2 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -36,7 +36,7 @@ int main(int argc, char** argv) { gmi_register_null(); gmi_register_mesh(); gmi_model* g = gmi_load(".null"); - apf::Mesh2* m = apf::loadMdsMesh(g,argv[1]); + apf::Mesh2* m = apf::loadMdsMesh(g,argv[1],&PCUObj); int dim = m->getDimension(); @@ -110,6 +110,6 @@ int main(int argc, char** argv) { SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/verify_convert.cc b/test/verify_convert.cc index 13f527f23..f312d202b 100644 --- a/test/verify_convert.cc +++ b/test/verify_convert.cc @@ -2,7 +2,6 @@ * this test verifies that the convert process properly clones the underlying * fields, numberings, and tags */ -#include #include #include #include @@ -12,14 +11,15 @@ #include #include #include -apf::Mesh2* createEmptyMesh() + +apf::Mesh2* createEmptyMesh(pcu::PCU *PCUObj) { gmi_model* mdl = gmi_load(".null"); - return apf::makeEmptyMdsMesh(mdl, 1, false); + return apf::makeEmptyMdsMesh(mdl, 1, false, PCUObj); } -apf::Mesh2* createMesh() +apf::Mesh2* createMesh(pcu::PCU *PCUObj) { - apf::Mesh2* m = createEmptyMesh(); + apf::Mesh2* m = createEmptyMesh(PCUObj); apf::Vector3 pts[2] = {apf::Vector3(0,0,0), apf::Vector3(1,0,0)}; apf::MeshEntity* verts[2]; for( int i=0; i<2; i++) @@ -43,13 +43,14 @@ class twox : public apf::Function { int main(int argc, char* argv[]) { MPI_Init(&argc, &argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_null(); // create meshes and write data to one of them - apf::Mesh* m1 = createMesh(); - apf::Mesh2* m2 = createEmptyMesh(); + apf::Mesh* m1 = createMesh(&PCUObj); + apf::Mesh2* m2 = createEmptyMesh(&PCUObj); // create field on m1 apf::Field* f = apf::createLagrangeField(m1, "field1", apf::SCALAR, 1); apf::Function* func = new twox(f); @@ -137,7 +138,7 @@ int main(int argc, char* argv[]) m2->end(it); // check that not transfering Fields/Numberings/Tags also works - apf::Mesh2* m3 = createEmptyMesh(); + apf::Mesh2* m3 = createEmptyMesh(&PCUObj); apf::convert(m1, m3, NULL, NULL, false); m3->verify(); @@ -160,7 +161,7 @@ int main(int argc, char* argv[]) apf::destroyMesh(m1); apf::destroyMesh(m2); apf::destroyMesh(m3); - PCU_Comm_Free(); + } MPI_Finalize(); return 0; } diff --git a/test/visualizeAnisoSizes.cc b/test/visualizeAnisoSizes.cc index 3f461860e..f27b25b7c 100644 --- a/test/visualizeAnisoSizes.cc +++ b/test/visualizeAnisoSizes.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX @@ -37,16 +36,18 @@ void visualizeSizeField( const char* frameName, int sampleSize[2], double userScale, - const char* outputPrefix); + const char* outputPrefix, + pcu::PCU *PCUObj); int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if (argc < 8) { - if (PCU_Comm_Self() == 0) { + if (PCUObj.Self() == 0) { printf("USAGE1: %s " " \n", argv[0]); printf("USAGE2: %s " @@ -77,7 +78,7 @@ int main(int argc, char** argv) int sampleSize[2] = {atoi(argv[5]),atoi(argv[6])}; double scale = atof(argv[7]); safe_mkdir(inPrefix); - visualizeSizeField(".null", meshFile, sizeName, frameName, sampleSize, scale, inPrefix); + visualizeSizeField(".null", meshFile, sizeName, frameName, sampleSize, scale, inPrefix, &PCUObj); #ifdef HAVE_SIMMETRIX gmi_sim_stop(); @@ -86,7 +87,7 @@ int main(int argc, char** argv) Sim_unregisterAllKeys(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } @@ -120,7 +121,7 @@ double getLargetsSize( maxSize = scales[2]; } m->end(it); - PCU_Max_Doubles(&maxSize, 1); + m->getPCU()->Max(&maxSize, 1); return maxSize; } @@ -131,19 +132,20 @@ void visualizeSizeField( const char* frameName, int sampleSize[2], double userScale, - const char* outputPrefix) + const char* outputPrefix, + pcu::PCU *PCUObj) { apf::Mesh2* m; #ifdef HAVE_SIMMETRIX /* if it is a simmetrix mesh */ if (ph::mesh_has_ext(meshFile, "sms")) { pParMesh sim_mesh = PM_load(meshFile, NULL, NULL); - m = apf::createMesh(sim_mesh); + m = apf::createMesh(sim_mesh, PCUObj); } else #endif { // load the mesh change to desired order and write as before vtks - m = apf::loadMdsMesh(modelFile,meshFile); + m = apf::loadMdsMesh(modelFile,meshFile,PCUObj); } m->verify(); diff --git a/test/viz.cc b/test/viz.cc index 8846e918d..cf99e58fa 100644 --- a/test/viz.cc +++ b/test/viz.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include "../viz/viz.h" @@ -37,17 +36,18 @@ int main(int argc, char** argv) int provided; MPI_Init_thread(&argc,&argv,MPI_THREAD_MULTIPLE,&provided); PCU_ALWAYS_ASSERT(provided==MPI_THREAD_MULTIPLE); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); Visualization v; char output[128]; - snprintf(output,128,"%d",PCU_Comm_Self()); + snprintf(output,128,"%d",PCUObj.Self()); std::string part_num(output); apf::MeshIterator* itr; @@ -105,6 +105,6 @@ int main(int argc, char** argv) v.breakpoint("Part Boundaries"); freeMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/vtxBalance.cc b/test/vtxBalance.cc index cca54d464..163330f39 100644 --- a/test/vtxBalance.cc +++ b/test/vtxBalance.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -31,10 +30,11 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -48,7 +48,7 @@ int main(int argc, char** argv) #endif gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); Parma_PrintPtnStats(m, "initial"); apf::MeshTag* weights = setVtxWeights(m); const double step = 0.5; const int verbose = 1; @@ -68,6 +68,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/vtxEdgeElmBalance.cc b/test/vtxEdgeElmBalance.cc index 7ce4b9ecc..b79239277 100644 --- a/test/vtxEdgeElmBalance.cc +++ b/test/vtxEdgeElmBalance.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #ifdef HAVE_SIMMETRIX #include @@ -25,7 +24,7 @@ namespace { apf::MeshTag* setWeights(apf::Mesh* m, double edgeWeight) { double weights[4] = {1.,edgeWeight,1.,1.}; - weights[1] = (!PCU_Comm_Self()) ? edgeWeight : 1.0; + weights[1] = (!m->getPCU()->Self()) ? edgeWeight : 1.0; const int d = m->getDimension(); apf::MeshTag* tag = m->createDoubleTag("parma_weight", 1); setWeight(m, tag, 0, weights[0]); @@ -46,10 +45,11 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 6); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 6 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -64,12 +64,12 @@ int main(int argc, char** argv) gmi_register_mesh(); //load model and mesh double targetImb = atof(argv[5]); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); apf::MeshTag* weights = setWeights(m,atof(argv[4])); Parma_PrintWeightedPtnStats(m, weights, "initial"); const double step = 0.5; const int verbose = 1; apf::Balancer* balancer = Parma_MakeVtxEdgeElmBalancer(m, step, verbose); - if( !PCU_Comm_Self() ) + if( !m->getPCU()->Self() ) fprintf(stderr, "STATUS target imbalance %.2f\n", targetImb); balancer->balance(weights, targetImb); delete balancer; @@ -86,6 +86,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/vtxElmBalance.cc b/test/vtxElmBalance.cc index 20d3eff8a..12a47e39c 100644 --- a/test/vtxElmBalance.cc +++ b/test/vtxElmBalance.cc @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -35,17 +34,18 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc == 4); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); Parma_PrintPtnStats(m, "initial"); apf::MeshTag* weights = setWeights(m); const double step = 0.5; const int verbose = 2; @@ -59,6 +59,6 @@ int main(int argc, char** argv) // destroy mds m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/vtxElmMixedBalance.cc b/test/vtxElmMixedBalance.cc index 54e1b7169..3aa55fe99 100644 --- a/test/vtxElmMixedBalance.cc +++ b/test/vtxElmMixedBalance.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -18,7 +17,8 @@ int main(int argc, char** argv) const char* modelFile = argv[1]; const char* meshFile = argv[2]; MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -28,7 +28,7 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile); + ma::Mesh* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); m->verify(); ma::localizeLayerStacks(m); m->verify(); @@ -41,7 +41,7 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/writeIPFieldTest.cc b/test/writeIPFieldTest.cc index 7a45027d7..8a0f60e01 100644 --- a/test/writeIPFieldTest.cc +++ b/test/writeIPFieldTest.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include @@ -10,15 +9,16 @@ int main(int argc, char** argv) { PCU_ALWAYS_ASSERT(argc==3); MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&pcu_obj); m->verify(); apf::createIPField(m, "Cauchy_Stress", apf::MATRIX, 1); apf::writeVtkFiles("mesh_with_IPField", m); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/writePart.cc b/test/writePart.cc index ee22d41fb..4d631ec0e 100644 --- a/test/writePart.cc +++ b/test/writePart.cc @@ -2,28 +2,28 @@ #include #include #include -#include #include #include int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU pcu_obj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !pcu_obj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); } gmi_register_mesh(); - apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2]); - if (PCU_Comm_Self() == atoi(argv[3])) + apf::Mesh2* m = apf::loadMdsMesh(argv[1], argv[2], &pcu_obj); + if (m->getPCU()->Self() == atoi(argv[3])) apf::writeMdsPart(m, argv[4]); m->destroyNative(); apf::destroyMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/writeVtxPtn.cc b/test/writeVtxPtn.cc index b9835cdb7..600519842 100644 --- a/test/writeVtxPtn.cc +++ b/test/writeVtxPtn.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #include @@ -18,10 +17,10 @@ namespace { apf::destroyMesh(m); } - void getConfig(int argc, char** argv) + void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -34,13 +33,14 @@ namespace { int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); gmi_register_mesh(); - getConfig(argc,argv); - apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile); + getConfig(argc,argv,&PCUObj); + apf::Mesh2* m = apf::loadMdsMesh(modelFile,meshFile,&PCUObj); Parma_WriteVtxPtn(m,argv[3]); freeMesh(m); - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/xgc_split.cc b/test/xgc_split.cc index 217b558fd..c359c7447 100644 --- a/test/xgc_split.cc +++ b/test/xgc_split.cc @@ -14,10 +14,10 @@ const char* meshFile = 0; const char* outFile = 0; int serial=0; -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU* PCUObj) { if (argc < 4) { - if (!pumi_rank() ) + if (!PCUObj->Self()) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -55,9 +55,10 @@ Migration* get_xgc_plan(pGeom g, pMesh m) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - pumi_start(); - - getConfig(argc,argv); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); + pumi_load_pcu(&PCUObj); + getConfig(argc,argv,&PCUObj); pGeom g = pumi_geom_load(modelFile); pMesh m; @@ -93,7 +94,7 @@ int main(int argc, char** argv) pumi_mesh_delete(m); - pumi_finalize(); + } MPI_Finalize(); } diff --git a/test/zbalance.cc b/test/zbalance.cc index b0fac0263..7409e5291 100644 --- a/test/zbalance.cc +++ b/test/zbalance.cc @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include // exit and EXIT_FAILURE @@ -20,10 +19,11 @@ int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); if ( argc != 4 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj.Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -38,7 +38,7 @@ int main(int argc, char** argv) gmi_register_null(); gmi_register_mesh(); //load model and mesh - apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2]); + apf::Mesh2* m = apf::loadMdsMesh(argv[1],argv[2],&PCUObj); apf::MeshTag* weights = Parma_WeighByMemory(m); apf::Balancer* balancer = makeZoltanBalancer(m, apf::GRAPH, apf::REPARTITION); balancer->balance(weights, 1.10); @@ -56,6 +56,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/test/zsplit.cc b/test/zsplit.cc index 50b8259fb..547201a9a 100644 --- a/test/zsplit.cc +++ b/test/zsplit.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include #ifdef HAVE_SIMMETRIX @@ -40,28 +39,20 @@ apf::Migration* getPlan(apf::Mesh* m) return plan; } -void switchToOriginals() +pcu::PCU* getGroupedPCU(pcu::PCU *PCUObj) { - int self = PCU_Comm_Self(); + int self = PCUObj->Self(); int groupRank = self / partitionFactor; int group = self % partitionFactor; MPI_Comm groupComm; MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm); - PCU_Switch_Comm(groupComm); + return new pcu::PCU(groupComm); } -void switchToAll() -{ - MPI_Comm prevComm = PCU_Get_Comm(); - PCU_Switch_Comm(MPI_COMM_WORLD); - MPI_Comm_free(&prevComm); - PCU_Barrier(); -} - -void getConfig(int argc, char** argv) +void getConfig(int argc, char** argv, pcu::PCU *PCUObj) { if ( argc != 5 ) { - if ( !PCU_Comm_Self() ) + if ( !PCUObj->Self() ) printf("Usage: %s \n", argv[0]); MPI_Finalize(); exit(EXIT_FAILURE); @@ -70,7 +61,7 @@ void getConfig(int argc, char** argv) meshFile = argv[2]; outFile = argv[3]; partitionFactor = atoi(argv[4]); - PCU_ALWAYS_ASSERT(partitionFactor <= PCU_Comm_Peers()); + PCU_ALWAYS_ASSERT(partitionFactor <= PCUObj->Peers()); } } @@ -78,7 +69,8 @@ void getConfig(int argc, char** argv) int main(int argc, char** argv) { MPI_Init(&argc,&argv); - PCU_Comm_Init(); + { + pcu::PCU PCUObj = pcu::PCU(MPI_COMM_WORLD); lion_set_verbosity(1); #ifdef HAVE_SIMMETRIX MS_init(); @@ -88,19 +80,22 @@ int main(int argc, char** argv) gmi_register_sim(); #endif gmi_register_mesh(); - getConfig(argc,argv); - bool isOriginal = ((PCU_Comm_Self() % partitionFactor) == 0); + getConfig(argc,argv,&PCUObj); + bool isOriginal = ((PCUObj.Self() % partitionFactor) == 0); gmi_model* g = 0; g = gmi_load(modelFile); apf::Mesh2* m = 0; apf::Migration* plan = 0; - switchToOriginals(); + pcu::PCU *groupedPCUObj = getGroupedPCU(&PCUObj); if (isOriginal) { - m = apf::loadMdsMesh(g, meshFile); + m = apf::loadMdsMesh(g, meshFile, groupedPCUObj); plan = getPlan(m); } - switchToAll(); - m = apf::repeatMdsMesh(m, g, plan, partitionFactor); + //used switchPCU here to load the mesh on the groupedPCU, perform tasks and then call repeatMdsMesh + //on the globalPCU + if(m != nullptr) m->switchPCU(&PCUObj); + delete groupedPCUObj; + m = apf::repeatMdsMesh(m, g, plan, partitionFactor, &PCUObj); Parma_PrintPtnStats(m, ""); m->writeNative(outFile); freeMesh(m); @@ -110,6 +105,6 @@ int main(int argc, char** argv) SimModel_stop(); MS_exit(); #endif - PCU_Comm_Free(); + } MPI_Finalize(); } diff --git a/zoltan/apfInterElement.cc b/zoltan/apfInterElement.cc index c8fe29252..7c7183e82 100644 --- a/zoltan/apfInterElement.cc +++ b/zoltan/apfInterElement.cc @@ -1,4 +1,3 @@ -#include #include "apfZoltan.h" #include #include @@ -41,16 +40,16 @@ static void packOtherGid(GlobalNumbering* gn, MeshEntity* s) Mesh* m = getMesh(gn); Copy other = getOtherSide(m, s); long gid = getElementGid(gn, getSideElement(m, s)); - PCU_COMM_PACK(other.peer, other.entity); - PCU_COMM_PACK(other.peer, gid); + m->getPCU()->Pack(other.peer, other.entity); + m->getPCU()->Pack(other.peer, gid); } static void unpackOtherGid(Mesh* m, MeshTag* t) { MeshEntity* s; - PCU_COMM_UNPACK(s); + m->getPCU()->Unpack(s); long gid; - PCU_COMM_UNPACK(gid); + m->getPCU()->Unpack(gid); m->setLongTag(s, t, &gid); } @@ -59,15 +58,15 @@ MeshTag* tagOpposites(GlobalNumbering* gn, const char* name) Mesh* m = getMesh(gn); MeshTag* t = m->createLongTag(name, 1); int sideDim = m->getDimension() - 1; - PCU_Comm_Begin(); + m->getPCU()->Begin(); MeshIterator* it = m->begin(sideDim); MeshEntity* e; while ((e = m->iterate(it))) if (hasOtherSide(m, e)) packOtherGid(gn, e); m->end(it); - PCU_Comm_Send(); - while (PCU_Comm_Receive()) + m->getPCU()->Send(); + while (m->getPCU()->Receive()) unpackOtherGid(m, t); return t; } diff --git a/zoltan/apfZoltan.cc b/zoltan/apfZoltan.cc index bb04828bb..8023211bb 100644 --- a/zoltan/apfZoltan.cc +++ b/zoltan/apfZoltan.cc @@ -8,7 +8,6 @@ #include "apfZoltan.h" #include "apfZoltanMesh.h" #include -#include #include namespace apf { @@ -25,18 +24,18 @@ class ZoltanSplitter : public Splitter virtual ~ZoltanSplitter() {} virtual Migration* split(MeshTag* weights, double tolerance, int multiple) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); Migration* plan = bridge.run(weights, tolerance, multiple); if (isSynchronous) { for (int i = 0; i < plan->count(); ++i) { MeshEntity* e = plan->get(i); int p = plan->sending(e); - p += PCU_Proc_Self() * multiple; + p += bridge.mesh->getPCU()->Self() * multiple; plan->send(e, p); } } - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!bridge.mesh->getPCU()->Self()) lion_oprint(1, "planned Zoltan split factor %d to target" " imbalance %f in %f seconds\n", multiple, tolerance, t1 - t0); return plan; @@ -55,15 +54,15 @@ class ZoltanBalancer : public Balancer virtual ~ZoltanBalancer() {} virtual void balance(MeshTag* weights, double tolerance) { - double t0 = PCU_Time(); + double t0 = pcu::Time(); Migration* plan = bridge.run(weights, tolerance, 1); - if (!PCU_Comm_Self()) + if (!bridge.mesh->getPCU()->Self()) lion_oprint(1, "planned Zoltan balance to target " "imbalance %f in %f seconds\n", - tolerance, PCU_Time() - t0); + tolerance, pcu::Time() - t0); bridge.mesh->migrate(plan); - double t1 = PCU_Time(); - if (!PCU_Comm_Self()) + double t1 = pcu::Time(); + if (!bridge.mesh->getPCU()->Self()) lion_oprint(1,"Zoltan balanced to %f in %f seconds\n", tolerance, t1-t0); } diff --git a/zoltan/apfZoltanCallbacks.cc b/zoltan/apfZoltanCallbacks.cc index b7a6316e6..63bed1064 100644 --- a/zoltan/apfZoltanCallbacks.cc +++ b/zoltan/apfZoltanCallbacks.cc @@ -9,7 +9,6 @@ #include "apfZoltanMesh.h" #include "apfZoltan.h" #include "apfShape.h" -#include #include #include #include @@ -273,7 +272,7 @@ ZoltanData::ZoltanData(ZoltanMesh* zb_) : zb(zb_) if (zb->isLocal) comm = MPI_COMM_SELF; else - comm = PCU_Get_Comm(); + comm = zb->mesh->getPCU()->GetMPIComm(); ztn =Zoltan_Create(comm); import_gids = NULL; import_lids = NULL; @@ -323,9 +322,10 @@ void ZoltanData::setup() Zoltan_Set_Param(ztn, "obj_weight_dim", paramStr); Zoltan_Set_Param(ztn, "edge_weight_dim", "0"); + Mesh* m = zb->mesh; //Debug snprintf(paramStr, 128, "%d", dbgLvl); - if ( zb->isLocal && 0 != PCU_Comm_Self() ) + if ( zb->isLocal && 0 != m->getPCU()->Self() ) snprintf(paramStr, 128, "%d", 0); //if local silence all but rank 0 Zoltan_Set_Param(ztn, "debug_level", paramStr); Zoltan_Set_Param(ztn, "PARMETIS_OUTPUT_LEVEL", paramStr); @@ -345,7 +345,7 @@ void ZoltanData::setup() if ( zb->isLocal ) { snprintf(paramStr, 128, "%d", zb->multiple); } else { - snprintf(paramStr, 128, "%d", zb->multiple*PCU_Proc_Peers()); + snprintf(paramStr, 128, "%d", zb->multiple*m->getPCU()->Peers()); } Zoltan_Set_Param(ztn, "NUM_GLOBAL_PARTS", paramStr); snprintf(paramStr, 128, "%d", zb->multiple); @@ -372,7 +372,7 @@ void ZoltanData::ptn() &import_to_part, &num_exported, &export_gids, &export_lids, &export_procs, &export_to_part); if( ZOLTAN_OK != ret ) { - if( 0 == PCU_Comm_Self() ) + if( 0 == zb->mesh->getPCU()->Self() ) lion_eprint(1, "ERROR Zoltan partitioning failed\n"); exit(EXIT_FAILURE); } diff --git a/zoltan/apfZoltanMesh.cc b/zoltan/apfZoltanMesh.cc index 09df9723e..853186022 100644 --- a/zoltan/apfZoltanMesh.cc +++ b/zoltan/apfZoltanMesh.cc @@ -8,7 +8,6 @@ #include "apfZoltanMesh.h" #include "apfZoltanCallbacks.h" #include "apfZoltan.h" -#include #include namespace apf {