Merge pull request #104 from Adrian-Diaz/Adrian's-Branch

Nested DO

examples/kokkos_for.cpp

@@ -441,10 +441,62 @@ int main(int argc, char* argv[])
             //    int jend = (j_j+1)*32;
             }, result);
             hierTest2D(i_i,j_j)= result;
-            //printf("value at %d , %d is %f\n", i_i, j_j, hierTest2D(i_i,j_j));
+            printf("value at %d , %d is %f\n", i_i, j_j, hierTest2D(i_i,j_j));
         });
     });
     Kokkos::fence();
+
+    printf("\n\n\nHierarchical DO Reduce\n");
+    //2D nesting
+    DO_FIRST(i_i,1,hiersize-1, {
+        // Kokkos::parallel_for( \
+        //Kokkos::TeamPolicy<>( 32, Kokkos::AUTO, 32 ), \
+        //KOKKOS_LAMBDA ( const Kokkos::TeamPolicy<>::member_type &teamMember ) {
+        //const int i_i = TEAM_ID;
+        double result = 0;
+        double lsum;
+        DO_REDUCE_SUM_SECOND(j_j, i_i, hiersize-1, lsum, {
+            lsum += hierTest3D(i_i,j_j,0);
+            // Kokkos::parallel_for( \
+            //Kokkos::TeamThreadRange( teamMember, istart, iend ), [&] ( const int (j_j) ) {
+            // hierTest2D(i_i,j_j) = i_i * (j_j+1);
+            //    int jstart = j_j*32;
+            //    int jend = (j_j+1)*32;
+        }, result);
+        hierTest1D(i_i)= result;
+        //printf("value at %d is %f\n", i_i, hierTest1D(i_i));
+    });
+    Kokkos::fence();
+
+    printf("\n\n\nHierarchical Vectorized DO Reduce\n");
+    //3D vector nesting
+    DO_FIRST(i_i,0,hiersize-1, {
+        // Kokkos::parallel_for( \
+        //Kokkos::TeamPolicy<>( 32, Kokkos::AUTO, 32 ), \
+        //KOKKOS_LAMBDA ( const Kokkos::TeamPolicy<>::member_type &teamMember ) {
+        //const int i_i = TEAM_ID;
+        double result = 0;
+        double lsum;
+        DO_SECOND(j_j, i_i, hiersize-1, {
+            // Kokkos::parallel_for( \
+            //Kokkos::TeamThreadRange( teamMember, istart, iend ), [&] ( const int (j_j) ) {
+            // hierTest2D(i_i,j_j) = i_i * (j_j+1);
+            //    int jstart = j_j*32;
+            //    int jend = (j_j+1)*32;
+            DO_REDUCE_SUM_THIRD(k_k, i_i, j_j-1, lsum, {
+            lsum += hierTest3D(i_i,j_j,k_k);
+            // Kokkos::parallel_for( \
+            //Kokkos::TeamThreadRange( teamMember, istart, iend ), [&] ( const int (j_j) ) {
+            // hierTest2D(i_i,j_j) = i_i * (j_j+1);
+            //    int jstart = j_j*32;
+            //    int jend = (j_j+1)*32;
+            }, result);
+            hierTest2D(i_i,j_j)= result;
+            printf("value at %d , %d is %f\n", i_i, j_j, hierTest2D(i_i,j_j));
+        });
+    });
+    Kokkos::fence();
+
     for (int ppp = 0; ppp < hiersize; ppp++) {
         //printf("%f\n", hierTest1D(ppp));
         // printf("%f\n", hierTest2D(3,ppp));

lib/.gitignore

@@ -0,0 +1 @@
+Trilinos

scripts/build-matar.sh

@@ -44,7 +44,13 @@ show_help() {
     echo "          linux                       A general linux machine (that does not use modules)"
     echo "          mac                         A Mac computer. This option does not allow for cuda and hip builds, and build_cores will be set to 1"
     echo " "
-    echo "      --build_cores                   The number of build cores to be used by make and make install commands. The default is 1" 
+    echo "      --build_cores                   The number of build cores to be used by make and make install commands. The default is 1"
+    echo " "
+    echo "      --trilinos                      Decides if Trilinos is available for certain MATAR functionality"
+    echo " "
+    echo "          disabled                    Trilinos is not being used"
+    echo "          enabled                     Trilinos will be linked with MATAR to enable relevant functionality"
+    echo " "
     return 1
 }
 
@@ -54,12 +60,14 @@ execution="examples"
 machine="linux"
 kokkos_build_type="serial"
 build_cores="1"
+trilinos="disabled"
 
 # Define arrays of valid options
 valid_build_action=("full-app" "set-env" "install-matar" "install-kokkos" "matar")
 valid_execution=("examples" "test" "benchmark")
 valid_kokkos_build_types=("none" "serial" "openmp" "pthreads" "cuda" "hip" "serial_mpi" "openmp_mpi" "cuda_mpi" "hip_mpi")
 valid_machines=("darwin" "chicoma" "linux" "mac")
+valid_trilinos=("disabled" "enabled")
 
 # Parse command line arguments
 for arg in "$@"; do
@@ -114,6 +122,16 @@ for arg in "$@"; do
                 return 1
             fi
             ;;
+        --trilinos=*)
+            option="${arg#*=}"
+            if [[ " ${valid_trilinos[*]} " == *" $option "* ]]; then
+                trilinos="$option"
+            else
+                echo "Error: Invalid --kokkos_build_type specified."
+                show_help
+                return 1
+            fi
+            ;;
         --help)
             show_help
             return 1
@@ -150,6 +168,7 @@ echo "Build action - ${build_action}"
 echo "Execution - ${execution}"
 echo "Kokkos backend - ${kokkos_build_type}"
 echo "make -j ${build_cores}"
+echo "Trilinos - ${trilinos}"
 
 cd "$( dirname "${BASH_SOURCE[0]}" )"
 
@@ -162,10 +181,13 @@ if [ "$build_action" = "full-app" ]; then
     if [ ! -d "${builddir}" ]
     then
         mkdir -p ${builddir}
-    fi    
-    source kokkos-install.sh ${kokkos_build_type}
-    source matar-install.sh ${kokkos_build_type}
-    source cmake_build_${execution}.sh ${kokkos_build_type}
+    fi
+
+    if [ "$trilinos" = "disabled" ]; then    
+        source kokkos-install.sh ${kokkos_build_type}
+    fi
+    source matar-install.sh ${kokkos_build_type} ${trilinos}
+    source cmake_build_${execution}.sh ${kokkos_build_type} ${trilinos}
 elif [ "$build_action" = "install-kokkos" ]; then
     source kokkos-install.sh ${kokkos_build_type}
 elif [ "$build_action" = "install-matar" ]; then

scripts/cmake_build_examples.sh

@@ -1,6 +1,7 @@
 #!/bin/bash -e
 
 kokkos_build_type="${1}"
+trilinos="${2}"
 
 if [ ! -d "${EXAMPLE_SOURCE_DIR}/phaseFieldMPI/heffte" ]
 then

scripts/cmake_build_test.sh

@@ -1,6 +1,7 @@
 #!/bin/bash -e
 
 kokkos_build_type="${1}"
+trilinos="${2}"
 
 if [ ! -d "${TEST_SOURCE_DIR}/googletest" ]
 then

scripts/matar-install.sh

@@ -1,6 +1,7 @@
 #!/bin/bash -e
 
 kokkos_build_type=${1}
+trilinos=${2}
 
 rm -rf ${MATAR_INSTALL_DIR}
 mkdir -p ${MATAR_BUILD_DIR} 

src/include/macros.h

@@ -475,6 +475,37 @@ Kokkos::parallel_reduce( \
                         Kokkos::ThreadVectorRange( teamMember, z0, z1 ), [&] ( const int (k), decltype(lsum) &(lsum) ) \
                         {fcn}, result )
 
+#define \
+DO_FIRST(i, x0, x1, fcn) \
+Kokkos::parallel_for( \
+                        Kokkos::TeamPolicy<>( x1-x0+1, Kokkos::AUTO, 32 ), \
+                        KOKKOS_LAMBDA ( const Kokkos::TeamPolicy<>::member_type &teamMember ) \
+                        { const int i = TEAM_ID + x0; fcn} )
+
+#define \
+DO_SECOND(j, y0, y1, fcn) \
+Kokkos::parallel_for( \
+                        Kokkos::TeamThreadRange( teamMember, y0, y1+1 ), [&] ( const int (j) ) \
+                        {fcn} )
+
+#define \
+DO_REDUCE_SUM_SECOND(j, y0, y1, lsum, fcn, result) \
+Kokkos::parallel_reduce( \
+                        Kokkos::TeamThreadRange( teamMember, y0, y1+1 ), [&] ( const int (j), decltype(lsum) &(lsum) ) \
+                        {fcn}, result )
+
+#define \
+DO_THIRD(k, z0, z1, fcn) \
+Kokkos::parallel_for( \
+                        Kokkos::ThreadVectorRange( teamMember, z0, z1+1 ), [&] ( const int (k) ) \
+                        {fcn} )
+
+#define \
+DO_REDUCE_SUM_THIRD(k, z0, z1, lsum, fcn, result) \
+Kokkos::parallel_reduce( \
+                        Kokkos::ThreadVectorRange( teamMember, z0, z1+1 ), [&] ( const int (k), decltype(lsum) &(lsum) ) \
+                        {fcn}, result )
+
 //Kokkos Initialize
 #define \
     MATAR_KOKKOS_INIT \