From f81875c4da513ba418ba8e4243bf580896e10ad0 Mon Sep 17 00:00:00 2001 From: Stefan Date: Mon, 28 Oct 2019 04:40:31 -0500 Subject: [PATCH] load balance input data --- Makefile | 1 - README.md | 4 +-- example/conReader.h | 6 ++-- example/example.c | 24 ++++++++++--- example/quality.h | 14 ++++---- src/genmap-comm.c | 4 +-- src/genmap-eigen.c | 35 +++++++------------ src/genmap-gslib.h | 11 ------ src/genmap-handle.c | 4 +-- src/genmap-impl.h | 40 ++++++---------------- src/genmap-lanczos.c | 25 +++----------- src/genmap-rsb.c | 31 +++++++++-------- src/genmap-types.h | 4 +-- src/genmap-vector.c | 3 -- src/genmap.c | 12 ++----- src/genmap.h | 78 ++++++++++-------------------------------- src/parRSB.c | 81 ++++++++++++++++++++++++++++++++++++++------ src/parrsb-binsort.c | 16 ++++----- 18 files changed, 177 insertions(+), 216 deletions(-) diff --git a/Makefile b/Makefile index 3f2c849d..397b2334 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,6 @@ CSRCS:= $(SRCDIR)/genmap.c \ $(SRCDIR)/genmap-eigen.c $(SRCDIR)/genmap-laplacian.c $(SRCDIR)/genmap-lanczos.c \ $(SRCDIR)/genmap-rsb.c \ $(SRCDIR)/parrsb-binsort.c \ - $(SRCDIR)/parrsb-histogram.c \ $(SRCDIR)/genmap-chelpers.c \ $(SRCDIR)/parRSB.c COBJS:=$(CSRCS:.c=.o) diff --git a/README.md b/README.md index 49f93347..b98dd2a2 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ make CC=mpicc GSLIBPATH=/build all ```sh cd example -mpirun -np 4 ./example case01.co2 +mpirun -np 4 ./example 2 case01.co2 ``` ## C Interface @@ -36,4 +36,4 @@ opt (in) ... Additional parameters (to use defaults set opt[0] = 0). comm (in) ... MPI Communicator (size determines number of partitions). ``` -Note, any initial distribution of mesh elements is valid but only MPI ranks with nel>0 will participate. +Note, any initial distribution of mesh elements is valid. diff --git a/example/conReader.h b/example/conReader.h index 87b389d5..56caf9f3 100644 --- a/example/conReader.h +++ b/example/conReader.h @@ -1,5 +1,7 @@ struct con { - int nv, nelg, nel; + int nv; + int nelg; + int nel; long long *el, *vl; }; @@ -35,7 +37,7 @@ int conRead(const char *fname, struct con *c, MPI_Comm comm) { } int nelr = nelgt / np; - for(i = 0; i < nelgt % np; ++i) if(np - i == myid) nelr++; + for(i = 0; i < nelgt % np; ++i) if(np-i == myid) nelr++; int nelr_; MPI_Scan(&nelr, &nelr_, 1, MPI_INT, MPI_SUM, comm); diff --git a/example/example.c b/example/example.c index 02e78b1a..e96b3c42 100644 --- a/example/example.c +++ b/example/example.c @@ -24,7 +24,6 @@ int main(int argc, char *argv[]) { struct crystal cr; struct array eList; elm_data *data; - struct con con; int ierr; int e, n, nel, nv; @@ -33,10 +32,27 @@ int main(int argc, char *argv[]) { MPI_Init(&argc, &argv); comm_init(&comm, MPI_COMM_WORLD); - ierr = conRead(argv[1], &con, comm.c); + int color = MPI_UNDEFINED; + + if (argc != 3) { + if(comm.id) printf("usage: ./example <#nread> \n"); + return EXIT_FAILURE; + } + + int nRead = atoi(argv[1]); + char* conFile = argv[2]; + + if (comm.id < nRead) color = 1; + MPI_Comm comm_read; + MPI_Comm_split(comm.c, color, 0, &comm_read); + + ierr = 0; + struct con con = {}; + if (color != MPI_UNDEFINED) ierr = conRead(conFile, &con, comm_read); if(ierr) goto quit; - nv = con.nv; + comm_bcast(&comm, &con.nv, sizeof(int), 0); + nv = con.nv; nel = con.nel; int *part = (int*) malloc(nel * sizeof(int)); @@ -52,7 +68,7 @@ int main(int argc, char *argv[]) { for(data = eList.ptr, e = 0; e < nel; ++e) { data[e].proc = part[e]; data[e].id = con.el[e]; - for(int n = 0; n < nv; ++n) { + for(n = 0; n < nv; ++n) { data[e].vtx[n] = con.vl[e * nv + n]; } } diff --git a/example/quality.h b/example/quality.h index a8ae5e49..11d3d709 100644 --- a/example/quality.h +++ b/example/quality.h @@ -25,13 +25,13 @@ void printPartStat(long long *vtx, int nel, int nv, comm_ext ce) { if(np == 1) return; numPoints = nel * nv; - data = (long long*) malloc(numPoints * sizeof(long long)); + data = (long long*) malloc((numPoints+1) * sizeof(long long)); for(i = 0; i < numPoints; i++) data[i] = vtx[i]; gsh = gs_setup(data, numPoints, &comm, 0, gs_pairwise, 0); pw_data_nmsg(gsh, &Nmsg); - Ncomm = (int *) malloc(Nmsg * sizeof(int)); + Ncomm = (int *) malloc((Nmsg+1) * sizeof(int)); pw_data_size(gsh, Ncomm); gs_free(gsh); @@ -62,7 +62,8 @@ void printPartStat(long long *vtx, int nel, int nv, comm_ext ce) { comm_allreduce(&comm, gs_int, gs_min, &nssMin, 1, &b); comm_allreduce(&comm, gs_int, gs_add, &nssSum, 1, &b); - nsSum = nsSum / Nmsg; + if(Nmsg) nsSum = nsSum / Nmsg; + else nsSum=0; comm_allreduce(&comm, gs_int, gs_add, &nsSum, 1, &b); nelMax = nel; @@ -81,10 +82,11 @@ void printPartStat(long long *vtx, int nel, int nv, comm_ext ce) { " Max volume: %d | Min volume: %d | Avg volume: %lf\n", nssMax, nssMin, (double)nssSum / np); printf( - " Max elements: %d | Min elements: %d | Balance: %lf\n", - nelMax, nelMin, (double)nelMax / nelMin); + " Max elements: %d | Min elements: %d\n", + nelMax, nelMin); fflush(stdout); } - + + free(Ncomm); comm_free(&comm); } diff --git a/src/genmap-comm.c b/src/genmap-comm.c index 78f72144..f2505522 100644 --- a/src/genmap-comm.c +++ b/src/genmap-comm.c @@ -1,7 +1,5 @@ #include "genmap-impl.h" -// -// GenmapComm -// + int GenmapCreateComm(GenmapComm *c, GenmapCommExternal ce) { GenmapMalloc(1, c); comm_init(&(*c)->gsComm, ce); diff --git a/src/genmap-eigen.c b/src/genmap-eigen.c index 2b97cf03..e5282946 100644 --- a/src/genmap-eigen.c +++ b/src/genmap-eigen.c @@ -2,12 +2,7 @@ #include #include -// -// Routines to calculate Eigenvalues/vectors. -// -// -// Linear solve for Symmetric Tridiagonal Matrix -// + int GenmapSymTriDiagSolve(GenmapVector x, GenmapVector b, GenmapVector alpha, GenmapVector beta) { @@ -39,9 +34,7 @@ int GenmapSymTriDiagSolve(GenmapVector x, GenmapVector b, GenmapDestroyVector(diag); return 0; } -// -// Inverse power iterations -// + int GenmapInvPowerIter(GenmapVector eVector, GenmapVector alpha, GenmapVector beta, GenmapVector init, GenmapInt iter) { assert(alpha->size == beta->size + 1); @@ -61,10 +54,10 @@ int GenmapInvPowerIter(GenmapVector eVector, GenmapVector alpha, GenmapCopyVector(x, init); GenmapInt j; for(j = 0; j < iter; j++) { - // Ay = x + /* Ay = x */ GenmapSymTriDiagSolve(y, x, alpha, beta); - // Normalize by inf-norm(y) + /* Normalize by inf-norm(y) */ if(j != iter - 1) { GenmapScalar lambda = 1.0 / GenmapNormVector(y, -1); GenmapScaleVector(y, y, lambda); @@ -81,16 +74,12 @@ int GenmapInvPowerIter(GenmapVector eVector, GenmapVector alpha, return 0; } -// -// Sign routine -// + GenmapScalar GenmapSign(GenmapScalar a, GenmapScalar b) { GenmapScalar m = 1.0 ? b >= 0. : -1.0; return fabs(a) * m; } -// -// Routine to find eigenvectors and values of tri-diagonal matrix -// + int GenmapTQLI(GenmapHandle h, GenmapVector diagonal, GenmapVector upper, GenmapVector **eVectors, GenmapVector *eValues) { assert(diagonal->size == upper->size + 1); @@ -104,9 +93,9 @@ int GenmapTQLI(GenmapHandle h, GenmapVector diagonal, GenmapVector upper, GenmapCopyVector(e, upper); e->data[n - 1] = 0.0; - // Create the vector to store eigenvalues + /* Create the vector to store eigenvalues */ GenmapCreateVector(eValues, n); - // Init to identity + /* Init to identity */ GenmapMalloc(n, eVectors); GenmapInt i; for(i = 0; i < n; i++) { @@ -121,7 +110,7 @@ int GenmapTQLI(GenmapHandle h, GenmapVector diagonal, GenmapVector upper, do { for(m = l; m < n - 1; m++) { GenmapScalar dd = fabs(d->data[m]) + fabs(d->data[m + 1]); - // Should use a tolerance for this check + /* Should use a tolerance for this check */ if(fabs(e->data[m]) / dd < GENMAP_DP_TOL) break; } @@ -172,13 +161,13 @@ int GenmapTQLI(GenmapHandle h, GenmapVector diagonal, GenmapVector upper, p = s * r; d->data[i + 1] = g + p; g = c * r - b; - // Find eigenvectors + /* Find eigenvectors */ for(k = 0; k < n; k++) { f = (*eVectors)[k]->data[i + 1]; (*eVectors)[k]->data[i + 1] = s * (*eVectors)[k]->data[i] + c * f; (*eVectors)[k]->data[i] = c * (*eVectors)[k]->data[i] - s * f; } - // Done with eigenvectors + /* Done with eigenvectors */ } if(r < GENMAP_DP_TOL && i >= l) continue; @@ -190,7 +179,7 @@ int GenmapTQLI(GenmapHandle h, GenmapVector diagonal, GenmapVector upper, } while(m != l); } - // Orthnormalize eigenvectors -- Just normalize? + /* Orthnormalize eigenvectors -- Just normalize? */ for(i = 0; i < n; i++) { for(j = 0; j < i; j++) { GenmapScalar tmp = (*eVectors)[i]->data[j]; diff --git a/src/genmap-gslib.h b/src/genmap-gslib.h index c5586814..33145421 100644 --- a/src/genmap-gslib.h +++ b/src/genmap-gslib.h @@ -1,17 +1,6 @@ #ifndef _GENMAP_GSLIB_H_ #define _GENMAP_GSLIB_H_ -// Data type sint/uint -// (defualt) int -// #define USE_LONG long -// #define USE_LONG_LONG long long - -// Data type slong/ulong -// (default) int -// #define GLOBAL_LONG long -// #define GLOBAL_LONG_LONG long long -// #define GLOBAL_LONG_LONG - #include "gslib.h" #if !defined(MPI) diff --git a/src/genmap-handle.c b/src/genmap-handle.c index 02256c05..c07156c2 100644 --- a/src/genmap-handle.c +++ b/src/genmap-handle.c @@ -1,7 +1,5 @@ #include "genmap-impl.h" -// -// GenmapHandle -// + GenmapElements GenmapGetElements(GenmapHandle h) { return (GenmapElements) h->elementArray.ptr; } diff --git a/src/genmap-impl.h b/src/genmap-impl.h index 866abe29..c76428a2 100644 --- a/src/genmap-impl.h +++ b/src/genmap-impl.h @@ -11,25 +11,19 @@ #endif #include "genmap.h" -// -// Fiedler fields -// + #define GENMAP_FIEDLER 0 #define GENMAP_GLOBALID 1 #define GENMAP_PROC 2 #define GENMAP_ORIGIN 3 -// -// GenmapComm -// + struct GenmapComm_private { struct comm gsComm; struct gs_data *verticesHandle; GenmapScalar *laplacianWeights; buffer buf; }; -// -// GenmapElements -// + struct GenmapElement_private { GenmapScalar fiedler; GenmapLong globalId; @@ -38,15 +32,11 @@ struct GenmapElement_private { GenmapInt proc; GenmapInt origin; }; -// -// GenmapElements: Create, Destroy -// + int GenmapCreateElements(GenmapElements *e); int GenmapDestroyElements(GenmapElements e); GenmapElements GenmapGetElements_default(GenmapHandle h); -// -// GenmapHandle -// + struct GenmapHandle_private { GenmapComm global; GenmapComm local; @@ -65,27 +55,19 @@ struct GenmapHandle_private { parRSBHistogram histogram; }; -// -// GenmapHandle: Create, Destroy -// + int GenmapCreateHandle(GenmapHandle h); int GenmapDestroyHandle(GenmapHandle h); -// -// GenmapVector -// + struct GenmapVector_private { GenmapInt size; GenmapScalar *data; }; -// -// Memory management routines -// + #define GenmapMalloc(n, p) GenmapMallocArray ((n), sizeof(**(p)), p) #define GenmapCalloc(n, p) GenmapCallocArray ((n), sizeof(**(p)), p) #define GenmapRealloc(n, p) GenmapReallocArray((n), sizeof(**(p)), p) -// -// Binsort -// + void GenmapFiedlerMinMax(GenmapHandle h, GenmapScalar *min, GenmapScalar *max); void GenmapGlobalIdMinMax(GenmapHandle h, GenmapLong *min, GenmapLong *max); GenmapInt GenmapSetFiedlerBin(GenmapHandle h); @@ -93,9 +75,7 @@ GenmapInt GenmapSetGlobalIdBin(GenmapHandle h); void GenmapAssignBins(GenmapHandle h, int field, buffer *buf0); void GenmapTransferToBins(GenmapHandle h, int field, buffer *buf0); void GenmapBinSort(GenmapHandle h, int field, buffer *buf0); -// -// HistoSort -// + void parRSBHistogramSort(GenmapHandle h,GenmapComm c,int field,buffer *buf0); struct parRSBHistogram_private { diff --git a/src/genmap-lanczos.c b/src/genmap-lanczos.c index e9da4282..79c73638 100644 --- a/src/genmap-lanczos.c +++ b/src/genmap-lanczos.c @@ -2,9 +2,8 @@ #include #include -// -// Orthogonalize by 1-vector (vector of all 1's) -// + +/* Orthogonalize by 1-vector (vector of all 1's) */ int GenmapOrthogonalizebyOneVector(GenmapHandle h, GenmapComm c, GenmapVector q1, GenmapLong n) { GenmapInt i; @@ -21,9 +20,7 @@ int GenmapOrthogonalizebyOneVector(GenmapHandle h, GenmapComm c, return 0; } -// -// Lanczos version used in Paul's original genmap code. -// + int GenmapLanczosLegendary(GenmapHandle h, GenmapComm c, GenmapVector f, GenmapInt niter, GenmapVector **rr, GenmapVector diag, GenmapVector upper) { @@ -47,13 +44,11 @@ int GenmapLanczosLegendary(GenmapHandle h, GenmapComm c, GenmapVector f, GenmapScalar tmp; GenmapInt lelt = GenmapGetNLocalElements(h); - // Store Local Laplacian weights GenmapCreateVector(&weights, lelt); GenmapCreateZerosVector(&p, lelt); GenmapCreateVector(&w, lelt); GenmapInitLaplacian(h, c, weights); - // Create vector r orthogonalizing init in 1-norm to (1,1,1...) GenmapCreateVector(&r, lelt); GenmapCopyVector(r, f); GenmapOrthogonalizebyOneVector(h, c, r, GenmapGetNGlobalElements(h)); @@ -63,7 +58,6 @@ int GenmapLanczosLegendary(GenmapHandle h, GenmapComm c, GenmapVector f, rtol = rnorm * eps; rni = 1.0 / rnorm; - // Allocate memory for q-vectors if(*rr == NULL) { GenmapMalloc((size_t)(niter + 1), rr); GenmapInt i; @@ -83,7 +77,6 @@ int GenmapLanczosLegendary(GenmapHandle h, GenmapComm c, GenmapVector f, GenmapAxpbyVector(p, p, beta, r, 1.0); GenmapOrthogonalizebyOneVector(h, c, p, GenmapGetNGlobalElements(h)); - // Multiplication by the laplacian GenmapLaplacian(h, c, p, weights, w); GenmapScaleVector(w, w, -1.0); @@ -133,9 +126,7 @@ int GenmapLanczosLegendary(GenmapHandle h, GenmapComm c, GenmapVector f, return iter; } -// -// Lanczos version in Introduction to Sci. Comp by Prof. Heath. -// + int GenmapLanczos(GenmapHandle h, GenmapComm c, GenmapVector init, GenmapInt iter, GenmapVector **q, GenmapVector alpha, GenmapVector beta) { @@ -154,7 +145,6 @@ int GenmapLanczos(GenmapHandle h, GenmapComm c, GenmapVector init, GenmapInt lelt = GenmapGetNLocalElements(h); - // Create vector q1 orthogonalizing init in 1-norm to (1,1,1...) GenmapCreateVector(&q1, lelt); GenmapCopyVector(q1, init); GenmapOrthogonalizebyOneVector(h, c, q1, GenmapGetNGlobalElements(h)); @@ -163,14 +153,12 @@ int GenmapLanczos(GenmapHandle h, GenmapComm c, GenmapVector init, normq1 = sqrt(normq1); GenmapScaleVector(q1, q1, 1. / normq1); - // Create vector u GenmapCreateVector(&u, lelt); - // Set q_0 and beta_0 to zero (both uses 0-indexing) + /* Set q_0 and beta_0 to zero (both uses 0-indexing) */ GenmapCreateZerosVector(&q0, lelt); beta->data[0] = 0.; - // Allocate memory for q-vectors if(*q == NULL) { GenmapMalloc((size_t)iter, q); GenmapInt i; @@ -178,18 +166,15 @@ int GenmapLanczos(GenmapHandle h, GenmapComm c, GenmapVector init, (*q)[i] = NULL; } - // Store Local Laplacian weights GenmapVector weights; GenmapCreateVector(&weights, lelt); GenmapInitLaplacian(h, c, weights); int k; for(k = 0; k < iter; k++) { - // Store q1 GenmapCreateVector(&(*q)[k], lelt); GenmapCopyVector((*q)[k], q1); - // Multiplication by the laplacian GenmapLaplacian(h, c, q1, weights, u); alpha->data[k] = GenmapDotVector(q1, u); diff --git a/src/genmap-rsb.c b/src/genmap-rsb.c index fdb85f33..80f1b5f4 100644 --- a/src/genmap-rsb.c +++ b/src/genmap-rsb.c @@ -6,7 +6,6 @@ #include int GenmapFiedler(GenmapHandle h, GenmapComm c, int maxIter, int global) { - // 1. Do lanczos in local communicator. GenmapInt lelt = GenmapGetNLocalElements(h); GenmapVector initVec, alphaVec, betaVec; @@ -17,10 +16,12 @@ int GenmapFiedler(GenmapHandle h, GenmapComm c, int maxIter, int global) { #if defined(GENMAP_PAUL) if(global > 0) { for(i = 0; i < lelt; i++) { - //if(GenmapGetLocalStartIndex(h) + i + 1 < GenmapGetNGlobalElements(h) / 2) - // initVec->data[i] = GenmapGetLocalStartIndex(h) + i + 1 + 1000. * - // GenmapGetNGlobalElements(h); - //else +/* + if(GenmapGetLocalStartIndex(h) + i + 1 < GenmapGetNGlobalElements(h) / 2) + initVec->data[i] = GenmapGetLocalStartIndex(h) + i + 1 + 1000. * + GenmapGetNGlobalElements(h); + else +*/ initVec->data[i] = GenmapGetLocalStartIndex(h) + i + 1; } } else { @@ -59,7 +60,7 @@ int GenmapFiedler(GenmapHandle h, GenmapComm c, int maxIter, int global) { GenmapCreateVector(&evTriDiag, iter); #if defined(GENMAP_PAUL) - // Use TQLI and find the minimum eigenvalue and associated vector + /* Use TQLI and find the minimum eigenvalue and associated vector */ GenmapVector *eVectors, eValues; GenmapTQLI(h, alphaVec, betaVec, &eVectors, &eValues); @@ -92,7 +93,6 @@ int GenmapFiedler(GenmapHandle h, GenmapComm c, int maxIter, int global) { GenmapDestroyVector(init); #endif - // Multiply tri-diagonal matrix by [q1, q2, ...q_{iter}] GenmapInt j; GenmapCreateZerosVector(&evLanczos, lelt); for(i = 0; i < lelt; i++) { @@ -121,7 +121,6 @@ int GenmapFiedler(GenmapHandle h, GenmapComm c, int maxIter, int global) { elements[i].fiedler = evLanczos->data[i]; } - // n. Destory the data structures GenmapDestroyVector(initVec); GenmapDestroyVector(alphaVec); GenmapDestroyVector(betaVec); @@ -173,13 +172,15 @@ void GenmapRSB(GenmapHandle h) { #if defined(GENMAP_PAUL) int global = 1; - //int global = (GenmapCommSize(GenmapGetLocalComm(h)) == GenmapCommSize( - // GenmapGetGlobalComm(h))); - //GenmapElements e = GenmapGetElements(h); - //GenmapScan(h, GenmapGetLocalComm(h)); - //for(i = 0; i < GenmapGetNLocalElements(h); i++) { - // e[i].globalId0 = GenmapGetLocalStartIndex(h) + i + 1; - //} +/* + int global = (GenmapCommSize(GenmapGetLocalComm(h)) == GenmapCommSize( + GenmapGetGlobalComm(h))); + GenmapElements e = GenmapGetElements(h); + GenmapScan(h, GenmapGetLocalComm(h)); + for(i = 0; i < GenmapGetNLocalElements(h); i++) { + e[i].globalId0 = GenmapGetLocalStartIndex(h) + i + 1; + } +*/ #else int global = (GenmapCommSize(GenmapGetLocalComm(h)) == GenmapCommSize( GenmapGetGlobalComm(h))); diff --git a/src/genmap-types.h b/src/genmap-types.h index 050c1838..264989e7 100644 --- a/src/genmap-types.h +++ b/src/genmap-types.h @@ -2,9 +2,7 @@ #define _GENMAP_TYPES_H_ #include -// -// Genmap types -// + typedef long long GenmapLong; typedef unsigned long long GenmapULong; #define GenmapLongFormat "%lld" diff --git a/src/genmap-vector.c b/src/genmap-vector.c index b414564d..2f0eef2d 100644 --- a/src/genmap-vector.c +++ b/src/genmap-vector.c @@ -5,9 +5,6 @@ #include #include -// -// Vector operations -// int GenmapCreateVector(GenmapVector *x, GenmapInt size) { /* Asserts: - size > 0 diff --git a/src/genmap.c b/src/genmap.c index 66822a05..6fa9a947 100644 --- a/src/genmap.c +++ b/src/genmap.c @@ -3,9 +3,7 @@ #include #include -// -// GenmapInit -// + int GenmapInit(GenmapHandle *h, GenmapCommExternal ce) { GenmapMalloc(1, h); GenmapHandle h_ = *h; @@ -22,9 +20,7 @@ int GenmapInit(GenmapHandle *h, GenmapCommExternal ce) { GenmapMalloc(1,&h_->histogram); return 0; } -// -// GenmapFinalize -// + int GenmapFinalize(GenmapHandle h) { if(GenmapGetGlobalComm(h)) GenmapDestroyComm(GenmapGetGlobalComm(h)); @@ -39,9 +35,7 @@ int GenmapFinalize(GenmapHandle h) { return 0; } -// -// GenmapMalloc, Realloc, Calloc and Free -// + int GenmapMallocArray(size_t n, size_t unit, void *p) { int ierr = posix_memalign((void **)p, GENMAP_ALIGN, n * unit); if(ierr) diff --git a/src/genmap.h b/src/genmap.h index 250a31f7..ca61138c 100644 --- a/src/genmap.h +++ b/src/genmap.h @@ -1,67 +1,41 @@ #ifndef _GENMAP_H_ #define _GENMAP_H_ -// -// Header for Genmap types -// + #include "genmap-types.h" -// -// Header for gslib -// #include "genmap-gslib.h" -// -// Header for MPI -// #include -// -// Genmap Operators -// + #define GENMAP_SUM 0 #define GENMAP_MAX 1 #define GENMAP_MIN 2 #define GENMAP_MUL 3 -// -// Genmap Memory Align -// + #define GENMAP_ALIGN 32 -// -// Genmap tolerances -// + #define GENMAP_SP_TOL 1e-05 #define GENMAP_DP_TOL 1e-12 #define GENMAP_TOL GENMAP_DP_TOL -// -// Genmap readers -// + #define GENMAP_READER_LEN 256 #define GENMAP_MAX_READERS 32 -// -// GenmapCommExternal -// + typedef MPI_Datatype GenmapDataType; typedef MPI_Comm GenmapCommExternal; -// -// Genmap Pointer types -// + typedef struct GenmapComm_private *GenmapComm; typedef struct GenmapHandle_private *GenmapHandle; typedef struct GenmapVector_private *GenmapVector; typedef struct GenmapElement_private *GenmapElements; typedef struct parRSBHistogram_private *parRSBHistogram; -// -// Genmap: Init, Finalize -// + int GenmapInit(GenmapHandle *h, GenmapCommExternal ce); int GenmapFinalize(GenmapHandle h); -// -// GenmapMalloc, Realloc, Calloc and Free -// + int GenmapMallocArray(size_t n, size_t unit, void *p); int GenmapCallocArray(size_t n, size_t unit, void *p); int GenmapReallocArray(size_t n, size_t unit, void *p); int GenmapFree(void *p); -// -// GenmapHandle Getters/Setters -// + GenmapElements GenmapGetElements(GenmapHandle h); void GenmapSetElements(GenmapHandle h, GenmapElements elements); @@ -84,9 +58,7 @@ int GenmapGetNVertices(GenmapHandle h); void GenmapSetNVertices(GenmapHandle, int nVertices); void GenmapScan(GenmapHandle h, GenmapComm c); -// -// GenmapComm -// + int GenmapCreateComm(GenmapComm *c, GenmapCommExternal ce); int GenmapCommSize(GenmapComm c); int GenmapCommRank(GenmapComm c); @@ -102,13 +74,9 @@ void GenmapSplitComm(GenmapHandle h, GenmapComm *c, int bin); int GenmapCrystalInit(GenmapHandle h, GenmapComm c); int GenmapCrystalTransfer(GenmapHandle h, int field); int GenmapCrystalFinalize(GenmapHandle h); -// -// Function to read/write from/to FILE -// + int GenmapRead(GenmapHandle h, void *data); -// -// GenmapVector operations -// + int GenmapCreateVector(GenmapVector *x, GenmapInt size); int GenmapSetVector(GenmapVector x, GenmapScalar *array); int GenmapGetVector(GenmapVector x, GenmapScalar *array); @@ -132,22 +100,16 @@ GenmapScalar GenmapNormVector(GenmapVector x, GenmapInt p); int GenmapPrintVector(GenmapVector x); int GenmapDestroyVector(GenmapVector x); -// -// Functions to do Laplacian of the dual graph -// + int GenmapInitLaplacian(GenmapHandle h, GenmapComm c, GenmapVector weights); int GenmapLaplacian(GenmapHandle h, GenmapComm c, GenmapVector u, GenmapVector weights, GenmapVector v); -// -// Eigenvalue/vector calculations -// + int GenmapInvPowerIter(GenmapVector eVector, GenmapVector alpha, GenmapVector beta, GenmapVector init, int iter); int GenmapTQLI(GenmapHandle h, GenmapVector diagonal, GenmapVector upper, GenmapVector **eVectors, GenmapVector *eValues); -// -// Lanczos routines -// + int GenmapOrthogonalizebyOneVector(GenmapHandle h, GenmapComm c, GenmapVector q1, GenmapLong n); int GenmapLanczosLegendary(GenmapHandle h, GenmapComm c, GenmapVector f, @@ -156,14 +118,10 @@ int GenmapLanczosLegendary(GenmapHandle h, GenmapComm c, GenmapVector f, int GenmapLanczos(GenmapHandle h, GenmapComm c, GenmapVector init, GenmapInt iter, GenmapVector **q, GenmapVector alpha, GenmapVector beta); -// -// Fiedler and rsb -// + int GenmapFiedler(GenmapHandle h, GenmapComm c, int maxIter, int global); void GenmapRSB(GenmapHandle h); -// -// Debug routines -// + double GenmapGetMaxRss(); void GenmapPrintStack(); #endif diff --git a/src/parRSB.c b/src/parRSB.c index a2b8d8ff..fa7b49cf 100644 --- a/src/parRSB.c +++ b/src/parRSB.c @@ -7,6 +7,14 @@ #include "genmap-impl.h" #include "parRSB.h" +#define MAXNV 8 /* maximum number of vertices per element */ +typedef struct { + int proc; + GenmapLong id; + int part; + long long vtx[MAXNV]; +} elm_data; + void fparRSB_partMesh(int *part, long long *vtx, int *nel, int *nve, int *options, int *comm, int *err) { *err = 1; @@ -18,14 +26,52 @@ void fparRSB_partMesh(int *part, long long *vtx, int *nel, int *nve, int parRSB_partMesh(int *part, long long *vtx, int nel, int nve, int *options, MPI_Comm comm) { - int bin = nel > 0; - int rank, size; - MPI_Comm_rank(comm, &rank); - MPI_Comm_size(comm, &size); + int rank,size; + MPI_Comm_rank(comm,&rank); + MPI_Comm_size(comm,&size); + + /* load balance input data */ + GenmapLong nelg; + GenmapLong nell = nel; + MPI_Allreduce(&nell,&nelg,1,MPI_LONG_LONG_INT,MPI_SUM,comm); + GenmapLong nstar = nelg/size; + if(nstar == 0) nstar = 1; + + GenmapLong nelg_start; + MPI_Scan(&nell,&nelg_start,1,MPI_LONG_LONG_INT,MPI_SUM,comm); + nelg_start-=nel; + + struct array eList; + elm_data *data; + + array_init(elm_data,&eList,nel), eList.n=nel; + int e, n; + for(data=eList.ptr,e=0; esize*nstar) data[e].proc= (eg%size)-1; + + for(n=0; n0, rank, &commRSB); - if(bin > 0) { + if(nel>0) { double time0 = comm_time(); GenmapHandle h; GenmapInit(&h, commRSB); @@ -55,7 +101,7 @@ int parRSB_partMesh(int *part, long long *vtx, int nel, int nve, int *options, for(i = 0; i < nel; i++) { e[i].origin = id; for(j = 0; j < nve; j++) { - e[i].vertices[j] = vtx[i * nve + j]; + e[i].vertices[j] = data[i].vtx[j]; } } @@ -70,17 +116,14 @@ int parRSB_partMesh(int *part, long long *vtx, int nel, int nve, int *options, GenmapCrystalTransfer(h, GENMAP_ORIGIN); GenmapCrystalFinalize(h); - // This should hold true assert(GenmapGetNLocalElements(h) == nel); e = GenmapGetElements(h); - buffer buf; buffer_init(&buf, 1024); sarray_sort(struct GenmapElement_private, e, (unsigned int)nel, globalId, TYPE_LONG, &buf); - buffer_free(&buf); for(i = 0; i < nel; i++) { - part[i] = e[i].proc; + data[i].part = e[i].proc; } if(id == 0 && h->dbgLevel > 0) @@ -92,5 +135,21 @@ int parRSB_partMesh(int *part, long long *vtx, int nel, int nve, int *options, MPI_Comm_free(&commRSB); + /* restore original input */ + sarray_transfer(elm_data,&eList,proc,0,&cr); + data=eList.ptr; + nel =eList.n; + sarray_sort(elm_data,data,(unsigned int)nel,id,TYPE_LONG,&buf); + MPI_Barrier(comm); + + for(e = 0; e < nel; e++) { + part[e]=data[e].part; + } + + array_free(&eList); + buffer_free(&buf); + crystal_free(&cr); + comm_free(&c); + return 0; } diff --git a/src/parrsb-binsort.c b/src/parrsb-binsort.c index 2bdeb29f..3a8f1540 100644 --- a/src/parrsb-binsort.c +++ b/src/parrsb-binsort.c @@ -99,19 +99,13 @@ void GenmapAssignBins(GenmapHandle h, int field, buffer *buf0) { GenmapElements elements = GenmapGetElements(h); GenmapInt lelt = GenmapGetNLocalElements(h); - if(field == GENMAP_FIEDLER) { // Fiedler - // sort locally according to Fiedler vector + if(field == GENMAP_FIEDLER) { sarray_sort(struct GenmapElement_private, elements, (GenmapUInt)lelt, fiedler, TYPE_DOUBLE, buf0); - //sarray_sort_2(struct GenmapElement_private, elements, (GenmapUInt)lelt, fiedler, - // TYPE_DOUBLE, globalId, TYPE_LONG, buf0); - // set the bin based on Fiedler vector GenmapSetFiedlerBin(h); } else if(GENMAP_GLOBALID) { - // sort locally according to globalId sarray_sort(struct GenmapElement_private, elements, (GenmapUInt)lelt, globalId0, TYPE_LONG, buf0); - // set the bin based on globalId GenmapSetGlobalIdBin(h); } } @@ -120,7 +114,7 @@ void GenmapTransferToBins(GenmapHandle h, int field, buffer *buf0) { GenmapElements elements = GenmapGetElements(h); GenmapInt lelt = GenmapGetNLocalElements(h); - if(field == GENMAP_FIEDLER) { // Fiedler + if(field == GENMAP_FIEDLER) { sarray_transfer(struct GenmapElement_private, &(h->elementArray), proc, 0, &(h->cr)); GenmapScan(h, GenmapGetLocalComm(h)); @@ -128,8 +122,10 @@ void GenmapTransferToBins(GenmapHandle h, int field, buffer *buf0) { lelt = GenmapGetNLocalElements(h); sarray_sort(struct GenmapElement_private, elements, (GenmapUInt)lelt, fiedler, TYPE_DOUBLE, buf0); - //sarray_sort_2(struct GenmapElement_private, elements, (GenmapUInt)lelt, fiedler, - // TYPE_DOUBLE, globalId0, TYPE_LONG, buf0); +/* + sarray_sort_2(struct GenmapElement_private, elements, (GenmapUInt)lelt, fiedler, + TYPE_DOUBLE, globalId0, TYPE_LONG, buf0); +*/ } else if(field == GENMAP_GLOBALID) { sarray_transfer(struct GenmapElement_private, &(h->elementArray), proc, 0, &(h->cr));