🎉 Merge branch 'master' into release, bringing 3D vector solver and s…

…ome fixes

# Conflicts:
#	.gitlab-ci.yml

README.md

@@ -148,12 +148,13 @@ flups_free(data);
 To use the solver, you first need to create a topology
 ```cpp
 int  axis      = 0;              // aligned along the first dimension
+int  lda       = 1;              // scalar field
 int  nglob[3]  = {64, 128, 64};  // global size of 64x64x64
 int  nproc[3]  = {2, 1, 3};      // 6 procs; 2 x 1 x 3
 bool isComplex = false;          // real data
 
 // no specific alignement => we put a value of 1
-FLUPS_Topology *topo = flups_topo_new(axis, nglob, nproc, isComplex, NULL, 1, MPI_COMM_WORLD);
+FLUPS_Topology *topo = flups_topo_new(axis, lda, nglob, nproc, isComplex, NULL, 1, MPI_COMM_WORLD);
 
 // define additional quantities
 double L = {1.0, 2.0, 1.0};
@@ -173,7 +174,7 @@ flups_setup(mysolver,false);
 
 To solve a field `rhs` that has been defined on the topology, use
 ```cpp
-flups_solve(mysolver,rhs, rhs, SRHS);
+flups_solve(mysolver,rhs, rhs);
 ```
 
 Then, destroy the solver and the created topology

make_arch/make.docker_valid_dbg

@@ -1,10 +1,10 @@
 CXX = mpic++
 CC = mpicc
-CCFLAGS := -O0 -g --debug -DVERBOSE -std=c99 -fopenmp -Wno-format-security -DPROF -Wcast-align
-CXXFLAGS := -O0 -g --debug -DVERBOSE -std=c++17 -Wno-format-security -fopenmp -DPROF -Wcast-align
+CCFLAGS := -O0 -g --debug -DVERBOSE -std=c99 -fopenmp -Wno-format-security -DPROF -Wcast-align -DDUMP_DBG
+CXXFLAGS := -O0 -g --debug -DVERBOSE -std=c++17 -Wno-format-security -fopenmp -DPROF -Wcast-align -DDUMP_DBG
 LDFLAGS += -fopenmp -lstdc++ -lm
 
 FFTW_LIB := /usr/lib/x86_64-linux-gnu
 HDF5_INC := /usr/include/hdf5/openmpi
 HDF5_LIB := /usr/lib/x86_64-linux-gnu
-HDF5_LIBNAME := -lhdf5_openmpi
+HDF5_LIBNAME := -lhdf5_openmpi

samples/compareP3DFFT++/main_compare++.cpp

@@ -135,7 +135,7 @@ int main(int argc, char *argv[]) {
 
     // create a real topology
     int FLUnmemIn[3],FLUnmemOUT[3];
-    Topology *topoIn      = new Topology(0, nglob, nproc, false, NULL, FLUPS_ALIGNMENT, comm);
+    Topology *topoIn      = new Topology(0, 1, nglob, nproc, false, NULL, FLUPS_ALIGNMENT, comm);
     const int  nprocOut[3] = {1, 2, 1};
     const int  nglobOut[3] = {17, 32, 64};
 
@@ -306,12 +306,12 @@ int main(int argc, char *argv[]) {
                 // double       xF   = (istartGlo[0] + i0 + .5 ) * h[0]; 
 
                 size_t id;
-                id    = localIndex(0, i0, i1, i2, 0, FLUnmemIn, 1);
+                id    = localIndex(0, i0, i1, i2, 0, FLUnmemIn, 1, 0);
                 rhsFLU[id] = sin((c_2pi / L[0] * f) * x) * sin((c_2pi / L[1] * f) * y) * sin((c_2pi / L[2] * f) * z);
 
 #ifndef SKIP_P3D                 
                 //p3d does not care of the size you allocate, juste fill the first isize elements
-                id =  localIndex(0, i0, i1, i2, 0, P3DnlocIN, 1);
+                id =  localIndex(0, i0, i1, i2, 0, P3DnlocIN, 1, 0);
                 rhsP3D[id] = sin((c_2pi / L[0] * f) * x) * sin((c_2pi / L[1] * f) * y) * sin((c_2pi / L[2] * f) * z);
 #endif
             }
@@ -462,7 +462,7 @@ void print_res(double *A, const Topology* topo) {
         for (int i2 = 0; i2 < topo->nloc(ax2); i2++) {
             for (int i1 = 0; i1 < topo->nloc(ax1); i1++) {
                 //local indexes start
-                const size_t id = localIndex(ax0, 0, i1, i2, ax0, nmem,2);
+                const size_t id = localIndex(ax0, 0, i1, i2, ax0, nmem, 2, 0);
                 for (int i0 = 0; i0 < topo->nloc(ax0); i0++) {
 
                     if (std::fabs(A[id+i0*2]) + std::fabs(A[id+i0*2 + 1]) > 1.25e-4) {
@@ -475,7 +475,7 @@ void print_res(double *A, const Topology* topo) {
         for (int i2 = 0; i2 < topo->nloc(ax2); i2++) {
             for (int i1 = 0; i1 < topo->nloc(ax1); i1++) {
                 //local indexes start
-                const size_t id = localIndex(ax0, 0, i1, i2, ax0, nmem,1);
+                const size_t id = localIndex(ax0, 0, i1, i2, ax0, nmem, 1, 0);
                 for (int i0 = 0; i0 < topo->nloc(ax0); i0++) {
 
                     if (std::fabs(A[id+i0]) > 1.25e-4) {

samples/compareP3DFFT/main_compare.cpp

@@ -51,7 +51,7 @@ int main(int argc, char *argv[]) {
 
     // create a real topology
     int FLUnmemIn[3],FLUnmemOUT[3];
-    Topology *topoIn      = new Topology(0, nglob, nproc, false, NULL, FLUPS_ALIGNMENT, comm);
+    Topology *topoIn      = new Topology(0, 1, nglob, nproc, false, NULL, FLUPS_ALIGNMENT, comm);
     const int  nprocOut[3] = {1, 2, 1};
     const int  nglobOut[3] = {17, 32, 64};
 
@@ -169,11 +169,11 @@ int main(int argc, char *argv[]) {
                 double       z    = (istartGlo[2] + i2 ) * h[2]; //+ 0.5
 
                 size_t id;
-                id    = localIndex(0, i0, i1, i2, 0, FLUnmemIn, 1);
+                id         = localIndex(0, i0, i1, i2, 0, FLUnmemIn, 1, 0);
                 rhsFLU[id] = sin((c_2pi / L[0] * f) * x) * sin((c_2pi / L[1] * f) * y) * sin((c_2pi / L[2] * f) * z);
 
                 //p3d does not care of the size you allocate, juste fill the first isize elements
-                id =  localIndex(0, i0, i1, i2, 0, isize, 1);
+                id =  localIndex(0, i0, i1, i2, 0, isize, 1, 0);
                 rhsP3D[id] = sin((c_2pi / L[0] * f) * x) * sin((c_2pi / L[1] * f) * y) * sin((c_2pi / L[2] * f) * z);
             }
         }
@@ -343,7 +343,7 @@ void print_res(double *A, const Topology* topo) {
         for (int i2 = 0; i2 < topo->nloc(ax2); i2++) {
             for (int i1 = 0; i1 < topo->nloc(ax1); i1++) {
                 //local indexes start
-                const size_t id = localIndex(ax0, 0, i1, i2, ax0, nmem,2);
+                const size_t id = localIndex(ax0, 0, i1, i2, ax0, nmem, 2, 0);
                 for (int i0 = 0; i0 < topo->nloc(ax0); i0++) {
 
                     if (std::fabs(A[id+i0*2]) + std::fabs(A[id+i0*2 + 1]) > 1.25e-4) {
@@ -356,7 +356,7 @@ void print_res(double *A, const Topology* topo) {
         for (int i2 = 0; i2 < topo->nloc(ax2); i2++) {
             for (int i1 = 0; i1 < topo->nloc(ax1); i1++) {
                 //local indexes start
-                const size_t id = localIndex(ax0, 0, i1, i2, ax0, nmem,1);
+                const size_t id = localIndex(ax0, 0, i1, i2, ax0, nmem, 1, 0);
                 for (int i0 = 0; i0 < topo->nloc(ax0); i0++) {
 
                     if (std::fabs(A[id+i0]) > 1.25e-4) {

samples/solve_advanced_C/main_solve_advanced.c

@@ -28,7 +28,7 @@ int main(int argc, char *argv[]) {
     MPI_Comm_rank(comm, &rank);
     MPI_Comm_size(comm, &comm_size);
 
-    //-----------mak--------------------------------------------------------------
+    //-------------------------------------------------------------------------
     //Definition of the problem
     //-------------------------------------------------------------------------
     const int     nglob[3] = {64, 64, 64};
@@ -37,7 +37,7 @@ int main(int argc, char *argv[]) {
 
     const double h[3] = {L[0] / nglob[0], L[1] / nglob[1], L[2] / nglob[2]};
 
-    const FLUPS_BoundaryType mybc[3][2] = {PER, PER,
+    FLUPS_BoundaryType mybc[3][2] = {PER, PER,
                                             PER, PER,
                                             PER, PER};
 
@@ -53,12 +53,12 @@ int main(int argc, char *argv[]) {
 
     // create a real topology
 
-    FLUPS_Topology *topoIn      = flups_topo_new(0, nglob, nproc, false, NULL, FLUPS_ALIGNMENT,comm);
+    FLUPS_Topology *topoIn      = flups_topo_new(0, 1, nglob, nproc, false, NULL, FLUPS_ALIGNMENT,comm);
     const int             nprocOut[3] = {1, 2, 1};
 
 
     // solver creation and init
-    FLUPS_Solver *mysolver = flups_init(topoIn, mybc, h, L);
+    FLUPS_Solver *mysolver = flups_init(topoIn, &mybc, h, L);
     flups_set_greenType(mysolver,CHAT_2);
     double* solFLU = flups_setup(mysolver, true);
 
@@ -86,7 +86,7 @@ int main(int argc, char *argv[]) {
     // //-------------------------------------------------------------------------
 
     printf("[FLUPS] topo IN glob : %d %d %d \n",flups_topo_get_nglob(topoIn,0),flups_topo_get_nglob(topoIn,1),flups_topo_get_nglob(topoIn,2));
-    printf("[FLUPS] topo IN loc : %d*%d*%d = %d (check: %d %d %d)\n",flups_topo_get_nmem(topoIn,0),flups_topo_get_nmem(topoIn,1),flups_topo_get_nmem(topoIn,2),flups_topo_get_memsize(topoIn),flups_topo_get_nglob(topoIn,0),flups_topo_get_nglob(topoIn,1),flups_topo_get_nglob(topoIn,2));
+    printf("[FLUPS] topo IN loc : %d*%d*%d = %ld (check: %d %d %d)\n",flups_topo_get_nmem(topoIn,0),flups_topo_get_nmem(topoIn,1),flups_topo_get_nmem(topoIn,2),flups_topo_get_memsize(topoIn),flups_topo_get_nglob(topoIn,0),flups_topo_get_nglob(topoIn,1),flups_topo_get_nglob(topoIn,2));
     printf("[FLUPS] topo OUT glob : %d %d %d \n",flups_topo_get_nglob(topoSpec,0),flups_topo_get_nglob(topoSpec,1),flups_topo_get_nglob(topoSpec,2));
     printf("[FLUPS] topo OUT loc  : nmem: %d*%d*%d complex?:%d (nloc: %d %d %d)  \n",flups_topo_get_nmem(topoSpec,0),flups_topo_get_nmem(topoSpec,1),flups_topo_get_nmem(topoSpec,2),flups_topo_get_isComplex(topoSpec),flups_topo_get_nloc(topoSpec,0),flups_topo_get_nloc(topoSpec,1),flups_topo_get_nloc(topoSpec,2));
 
@@ -109,7 +109,7 @@ int main(int argc, char *argv[]) {
                 double       z    = (istartIn[2] + i2 + 0.5) * h[2];
 
                 size_t id;
-                id    = flups_locID(0, i0, i1, i2, 0, nmemIn, 1);
+                id    = flups_locID(0, i0, i1, i2, 0, 0, nmemIn, 1);
                 rhsFLU[id] = sin((c_2pi / L[0] * f) * x) * sin((c_2pi / L[1] * f) * y) * sin((c_2pi / L[2] * f) * z);
             }
         }
@@ -136,7 +136,7 @@ int main(int argc, char *argv[]) {
     // //-------------------------------------------------------------------------
     // /** - Multiplication in spectral space */
     // //-------------------------------------------------------------------------
-    flups_do_mult(mysolver,solFLU,SRHS);
+    flups_do_mult(mysolver,solFLU);
 
     // //-------------------------------------------------------------------------
     // /** - Gaussian filtering of the solution */
@@ -160,7 +160,7 @@ int main(int argc, char *argv[]) {
     for (int i2 = 0; i2 < flups_topo_get_nloc(topoSpec,ax2); i2++) {
         for (int i1 = 0; i1 < flups_topo_get_nloc(topoSpec,ax1); i1++) {
             //local indexes start
-            const size_t id = flups_locID(ax0, 0, i1, i2, ax0, nmemSpec,nf);
+            const size_t id = flups_locID(ax0, 0, i1, i2, 0, ax0, nmemSpec,nf);
             for (int i0 = 0; i0 < flups_topo_get_nloc(topoSpec,ax0); i0++) {
                 int is[3];
                 flups_symID(ax0, i0, i1, i2, istartSpec, symstart, 0, is);
@@ -237,7 +237,7 @@ void print_res(double *A, const FLUPS_Topology* topo) {
         for (int i2 = 0; i2 < flups_topo_get_nloc(topo,ax2); i2++) {
             for (int i1 = 0; i1 < flups_topo_get_nloc(topo,ax1); i1++) {
                 //local indexes start
-                const size_t id = flups_locID(ax0, 0, i1, i2, ax0, nmem,2);
+                const size_t id = flups_locID(ax0, 0, i1, i2, 0, ax0, nmem,2);
                 for (int i0 = 0; i0 < flups_topo_get_nloc(topo,ax0); i0++) {
                     printf("(%d %d %d) %lg +i1* %lg\n", i0 + gstart[ax0], i1 + gstart[ax1], i2 + gstart[ax2], A[id + i0 * 2], A[id + i0 * 2 + 1]);
                 }
@@ -247,7 +247,7 @@ void print_res(double *A, const FLUPS_Topology* topo) {
         for (int i2 = 0; i2 < flups_topo_get_nloc(topo,ax2); i2++) {
             for (int i1 = 0; i1 < flups_topo_get_nloc(topo,ax1); i1++) {
                 //local indexes start
-                const size_t id = flups_locID(ax0, 0, i1, i2, ax0, nmem,1);
+                const size_t id = flups_locID(ax0, 0, i1, i2, 0, ax0, nmem,1);
                 for (int i0 = 0; i0 < flups_topo_get_nloc(topo,ax0); i0++) {
                     printf("(%d %d %d) %lg \n", i0 + gstart[ax0], i1 + gstart[ax1], i2 + gstart[ax2], A[id + i0]);
                 }

samples/validation/data_ref/validation_3d_330400_typeGreen=0.txt

@@ -1 +1,2 @@
 8 7.204991050832e-02 2.404758239295e-01
+17 1.591082153721e-02 7.774850451306e-02 

samples/validation/data_ref/validation_3d_331400_typeGreen=0.txt

@@ -1 +1,2 @@
 8 7.204889841459e-02 2.404771560012e-01
+17 1.590620692368e-02 7.774864933693e-02 

samples/validation/data_ref/validation_3d_331411_typeGreen=0.txt

@@ -1,3 +1,4 @@
 8 2.556243887413e-01 8.031750419599e-01
 16 4.964407501836e-02 1.624461025199e-01
 32 1.271294793498e-02 5.581405355575e-02
+17 4.413576773139e-02 2.044117431074e-01 

samples/validation/data_ref/validation_3d_334144_typeGreen=0.txt

@@ -1 +1,2 @@
 8 6.920041901772e-03 3.889462539948e-02
+17 2.443562422060e-03 2.162738690023e-02 

samples/validation/data_ref/validation_3d_440411_typeGreen=0.txt

@@ -1 +1,2 @@
 8 1.519683722224e-02 4.951158955716e-02
+17 7.685652901145e-03 5.781401338986e-02 

samples/validation/data_ref/validation_3d_441400_typeGreen=0.txt

@@ -1 +1,2 @@
 8 8.920688166757e-03 3.558523190847e-02
+17 3.736157428473e-03 3.139269088664e-02 

samples/validation/data_ref/validation_3d_441411_typeGreen=0.txt

@@ -1 +1,2 @@
 8 1.525495739537e-02 4.951370235226e-02
+17 7.685310052641e-03 5.781399880372e-02 

samples/validation/run/juwels_kernel_valid.sh

@@ -16,24 +16,29 @@ module purge
 #module load IntelMPI/2018.5.288
 #module load IntelMPI/2019.3.199
 
-module load intel-para/2019a
-
+#module load intel-para/2019a
+##CHANGING COMMUNICATION METHOD, OTHERWISE MPI USE TOO MUCH MEMORY !
+#export PSP_UCP=1
+#export UCX_TLS=ud_mlx5,self,sm
+##export PSI_LOGGERDEBUG=1
+##export PSI_FORWARDERDEBUG=1
+
+#OLD DEVELOPMENT IMPI
 # module use /p/software/juwels/otherstages/
 # module load Stages/Devel-2019a
 # module load Intel
 # module load IntelMPI/2019.6.RC20191024
-
+#----------------------
+module use $OTHERSTAGES
+module load Stages/Devel-2019a
+module load Intel
+module load IntelMPI/2019.6.154
 
 module load FFTW/3.3.8
 module load HDF5/1.10.5
 module load METIS/5.1.0
 module list
 
-#CHANGING COMMUNICATION METHOD, OTHERWISE MPI USE TOO MUCH MEMORY !
-export PSP_UCP=1
-export UCX_TLS=ud_mlx5,self,sm
-#export PSI_LOGGERDEBUG=1
-#export PSI_FORWARDERDEBUG=1
 
 echo "----------------- launching job -----------------"
 echo "launch command: srun --label ${EXEC_FLUPS} -np ${MY_NX} ${MY_NY} ${MY_NZ} -res ${SIZE_X} ${SIZE_Y} ${SIZE_Z} -L ${L_X} ${L_Y} ${L_Z} -nres 1 -ns 20 -k 0"

samples/validation/run/juwels_strongscaling.sh

@@ -2,6 +2,7 @@
 
 HOME_FLUPS=/p/project/prpa79/flups/samples/validation
 KERNEL=juwels_kernel_valid.sh
+SCRATCH_FLDR=/p/scratch/prpa79/$(whoami)_impi19.6
 
 ## fixed parameters
 export INITIAL_SIZE_X=1152
@@ -38,7 +39,7 @@ export PARTITION=batch
 #-----------------------------------------------------------
 export EXEC_FLUPS=flups_validation_a2a
 
-SCRATCH=/p/scratch/prpa79/$(whoami)/flups_strong_a2a_${ver}
+SCRATCH=${SCRATCH_FLDR}/flups_strong_a2a_${ver}
 
 # clean the validation dir
 # rm -rf ${SCRATCH}
@@ -134,7 +135,7 @@ sbatch --ntasks=${MY_NTASKS} --cpus-per-task=${MY_NTHREADS} --time=${WT}  --part
 #-----------------------------------------------------------
 export EXEC_FLUPS=flups_validation_nb
 
-SCRATCH=/p/scratch/prpa79/$(whoami)/flups_strong_nb_${ver}
+SCRATCH=${SCRATCH_FLDR}/flups_strong_nb_${ver}
 
 # clean the validation dir
 # rm -rf ${SCRATCH}
@@ -241,7 +242,7 @@ export PARTITION=batch
 # #-----------------------------------------------------------
 export EXEC_FLUPS=flups_validation_a2a
 
-SCRATCH=/p/scratch/prpa79/$(whoami)/flups_strong_a2a_${ver}
+SCRATCH=${SCRATCH_FLDR}/flups_strong_a2a_${ver}
 
 # clean the validation dir
 # rm -rf ${SCRATCH}
@@ -313,7 +314,7 @@ sbatch --ntasks=${MY_NTASKS} --cpus-per-task=${MY_NTHREADS} --time=${WT}  --part
 #-----------------------------------------------------------
 export EXEC_FLUPS=flups_validation_nb
 
-SCRATCH=/p/scratch/prpa79/$(whoami)/flups_strong_nb_${ver}
+SCRATCH=${SCRATCH_FLDR}/flups_strong_nb_${ver}
 
 # clean the validation dir
 # rm -rf ${SCRATCH}
@@ -387,14 +388,14 @@ elif [ "$ver" = "Xlarge" ]; then
 ############################################################
 ############################################################
 ############################################################
-export PARTITION=large
+export PARTITION=batch
 
 # ############################################################
 # # ALL TO ALL
 # #-----------------------------------------------------------
 export EXEC_FLUPS=flups_validation_a2a
 
-SCRATCH=/p/scratch/prpa79/$(whoami)/flups_strong_a2a_${ver}
+SCRATCH=${SCRATCH_FLDR}/flups_strong_a2a_${ver}
 
 # clean the validation dir
 # rm -rf ${SCRATCH}
@@ -466,7 +467,7 @@ sbatch --ntasks=${MY_NTASKS} --cpus-per-task=${MY_NTHREADS} --time=${WT}  --part
 #-----------------------------------------------------------
 export EXEC_FLUPS=flups_validation_nb
 
-SCRATCH=/p/scratch/prpa79/$(whoami)/flups_strong_nb_${ver}
+SCRATCH=${SCRATCH_FLDR}/flups_strong_nb_${ver}
 
 # clean the validation dir
 # rm -rf ${SCRATCH}