Feat halo

.circleci/config.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - checkout
       - run: bash dash/scripts/dash-ci.sh Release
-      - run: bash -c 'export DART_FORCE_C_STD=99; dash/scripts/dash-ci.sh Minimal'
+      #- run: bash -c 'export DART_FORCE_C_STD=99; dash/scripts/dash-ci.sh Minimal'
       - run:
           command: bash dash/scripts/circleci/collect-artifacts.sh
           when: always
@@ -36,7 +36,7 @@ jobs:
     steps:
       - checkout
       - run: bash dash/scripts/dash-ci.sh Release
-      - run: bash -c 'export DART_FORCE_C_STD=99; dash/scripts/dash-ci.sh Minimal'
+      #- run: bash -c 'export DART_FORCE_C_STD=99; dash/scripts/dash-ci.sh Minimal'
       - run:
           command: bash dash/scripts/circleci/collect-artifacts.sh
           when: always
@@ -59,7 +59,7 @@ jobs:
     steps:
       - checkout
       - run: bash dash/scripts/dash-ci.sh Release
-      - run: bash -c 'export DART_FORCE_C_STD=99; dash/scripts/dash-ci.sh Minimal'
+      #- run: bash -c 'export DART_FORCE_C_STD=99; dash/scripts/dash-ci.sh Minimal'
       - run:
           command: bash dash/scripts/circleci/collect-artifacts.sh
           when: always
@@ -81,7 +81,7 @@ jobs:
     steps:
       - checkout
       - run: bash dash/scripts/dash-ci.sh Release
-      - run: bash -c 'export DART_FORCE_C_STD=99; dash/scripts/dash-ci.sh Minimal'
+      #- run: bash -c 'export DART_FORCE_C_STD=99; dash/scripts/dash-ci.sh Minimal'
       - run:
           command: bash dash/scripts/circleci/collect-artifacts.sh
           when: always

dart-impl/mpi/src/dart_globmem.c

@@ -24,6 +24,8 @@
 
 #include <stdio.h>
 #include <mpi.h>
+#include <stdlib.h>
+#include <unistd.h>
 
 /* For PRIu64, uint64_t in printf */
 #define __STDC_FORMAT_MACROS
@@ -225,6 +227,13 @@ dart_team_memalloc_aligned_dynamic(
     dtype_size, nelem, nbytes);
 
 
+  /**
+   * Work around a bug in Open MPI where registrations are expected to be page
+   * aligned: https://github.com/open-mpi/ompi/issues/7384
+   */
+  size_t page_size = sysconf(_SC_PAGE_SIZE);
+  nbytes = (nbytes + ((page_size)-1)) & ~((page_size)-1);
+
   dart_team_data_t *team_data = dart_adapt_teamlist_get(teamid);
   if (team_data == NULL) {
     DART_LOG_ERROR(
@@ -240,7 +249,7 @@ dart_team_memalloc_aligned_dynamic(
 #if !defined(DART_MPI_DISABLE_SHARED_WINDOWS)
 
   char     ** baseptr_set = NULL;
-	/* Allocate shared memory on sharedmem_comm, and create the related
+  /* Allocate shared memory on sharedmem_comm, and create the related
    * sharedmem_win */
   /* NOTE:
    * Windows should definitely be optimized for the concrete value type i.e.
@@ -331,15 +340,23 @@ dart_team_memalloc_aligned_dynamic(
     } else {
       baseptr_set[i] = sub_mem;
     }
-	}
-#else
-	if (MPI_Alloc_mem(nbytes, MPI_INFO_NULL, &sub_mem) != MPI_SUCCESS) {
+  }
+#else // DART_MPI_DISABLE_SHARED_WINDOWS
+#ifdef DYNAMIC_MEM_USE_POSIX_MEMALIGN
+  if (0 != posix_memalign((void**)&sub_mem, page_size, nbytes)) {
+    DART_LOG_ERROR(
+      "dart_team_memalloc_aligned_dynamic: bytes:%lu posix_memalign failed",
+      nbytes);
+  }
+#else // DYNAMIC_MEM_USE_POSIX_MEMALIGN
+  if (MPI_Alloc_mem(nbytes, MPI_INFO_NULL, &sub_mem) != MPI_SUCCESS) {
     DART_LOG_ERROR(
       "dart_team_memalloc_aligned_dynamic: bytes:%lu MPI_Alloc_mem failed",
       nbytes);
     return DART_ERR_OTHER;
   }
-#endif
+#endif // DYNAMIC_MEM_USE_POSIX_MEMALIGN
+#endif // DART_MPI_DISABLE_SHARED_WINDOWS
 
   MPI_Aint disp;
   MPI_Win  win = team_data->window;

dash/examples/ex.02.matrix.halo.heat_equation/main.cpp

@@ -123,26 +123,9 @@ int main(int argc, char *argv[])
 
     // Update Halos asynchroniously
     current_halo->update_async();
-
-    // optimized calculation of inner matrix elements
-    auto* current_begin = current_matrix.lbegin();
+
     auto* new_begin = new_matrix.lbegin();
-#if 0
-    for (auto i = inner_start; i < inner_end; i += offset) {
-      auto* center = current_begin + i;
-      auto* center_y_plus = center + offset;
-      auto* center_y_minus = center - offset;
-      for (auto j = 0; j < offset - 2;
-           ++j, ++center, ++center_y_plus, ++center_y_minus) {
-        /*auto dtheta =
-            (*(center - 1) + *(center + 1) - 2 * (*center)) / (dx * dx) +
-            (*(center_y_minus) + *(center_y_plus) - 2 * (*center)) / (dy * dy);
-        *(new_begin + i + j) = *center + k * dtheta * dt;*/
-        *(new_begin + i + j) = calc(center, center_y_minus, center_y_plus, center - 1, center + 1);
-      }
-    }
-#endif
-    // slow version
+
     auto it_end = current_op->inner.end();
     for(auto it = current_op->inner.begin(); it != it_end; ++it)
     {

dash/include/dash/Dimensional.h

@@ -441,6 +441,9 @@ class ViewSpec
   typedef ViewRegion<NumDimensions, IndexType> region_type;
   typedef ViewRange<IndexType>                 range_type;
 
+  using index_type = IndexType;
+  using size_type  = SizeType;
+
 public:
   template<dim_t NDim_, typename IndexType_>
   friend std::ostream& operator<<(
@@ -520,6 +523,8 @@ class ViewSpec
    */
   self_t & operator=(self_t && other)      = default;
 
+  static constexpr auto ndim() { return NumDimensions; }
+
   /**
    * Equality comparison operator.
    */

dash/include/dash/Matrix.h

@@ -116,11 +116,6 @@ template <
   typename LocalMemSpaceT>
 class LocalMatrixRef;
 
-namespace halo {
-  template<class _MatrixT>
-  class HaloMatrixWrapper;
-}
-
 /**
  * An n-dimensional array supporting subranges and sub-dimensional
  * projection.
@@ -167,8 +162,6 @@ class Matrix
   typedef PatternT
     Pattern_t;
 
-  template<class MatrixT_>
-  friend class halo::HaloMatrixWrapper;
 public:
   typedef GlobStaticMem<LocalMemSpaceT>
     GlobMem_t;

dash/include/dash/TeamSpec.h

@@ -307,7 +307,20 @@ class TeamSpec :
       }
       ++d;
     }
-    return this->at(neighbor_coords);
+    return team_unit_t(static_cast<dart_unit_t>(this->at(neighbor_coords)));
+  }
+
+  team_unit_t neighbor(const std::array<int, MaxDimensions>&  offsets) const
+  {
+    auto neighbor_coords = this->coords(_myid);
+    for (dim_t d = 0; d < MaxDimensions; ++d) {
+      neighbor_coords[d] += offsets[d];
+      if (neighbor_coords[d] < 0 ||
+          neighbor_coords[d] >= this->_extents[d]) {
+        return UNDEFINED_TEAM_UNIT_ID;
+      }
+    }
+    return team_unit_t(static_cast<dart_unit_t>(this->at(neighbor_coords)));
   }
 
   /**
@@ -340,13 +353,29 @@ class TeamSpec :
     dim_t d = 0;
     for (auto offset_d : offsets) {
       neighbor_coords[d] += offset_d;
-      if (neighbor_coords[d] < 0 ||
-          neighbor_coords[d] >= this->_extents[d]) {
+      if (neighbor_coords[d] < 0) {
+        neighbor_coords[d] = this->_extents[d] + offset_d;
+      } else if(neighbor_coords[d] >= this->_extents[d]) {
         neighbor_coords[d] %= this->_extents[d];
       }
       ++d;
     }
-    return at(neighbor_coords);
+    return team_unit_t(static_cast<dart_unit_t>(this->at(neighbor_coords)));
+  }
+
+  team_unit_t periodic_neighbor(const std::array<int, MaxDimensions>& offsets) const
+  {
+    auto neighbor_coords = this->coords(_myid);
+
+    for (dim_t d = 0; d < MaxDimensions; ++d) {
+      neighbor_coords[d] += offsets[d];
+      if (neighbor_coords[d] < 0) {
+        neighbor_coords[d] = this->_extents[d] + offsets[d];
+      } else if(neighbor_coords[d] >= this->_extents[d]) {
+        neighbor_coords[d] %= this->_extents[d];
+      }
+    }
+    return team_unit_t(static_cast<dart_unit_t>(this->at(neighbor_coords)));
   }
 
   /**