ch3: add ch3_vcrt to MPIR_Group

Group are a natural place to host vcrt (virtual connection reference
table). When communicators are duplicated, groups are simply inherited
and reference counted. Thus we won't end up with duplication of vcrt.

src/mpi/group/grouputil.c

@@ -83,12 +83,24 @@ int MPIR_Group_init(void)
     pmap->u.stride.stride = 1;
     pmap->u.stride.blocksize = 1;
 
+#ifdef MPID_DEV_GROUP_DECL
+    for (int i = 0; i < MPIR_GROUP_N_BUILTIN; i++) {
+        MPID_Group_init_hook(MPIR_Group_builtin + i);
+    }
+#endif
     return mpi_errno;
 }
 
 int MPIR_Group_finalize(void)
 {
     num_worlds = 0;
+
+    /* The builtin groups are never released. Make sure to call device hooks */
+#ifdef MPID_DEV_GROUP_DECL
+    MPID_Group_free_hook(MPIR_Group_builtin + 1);       /* MPIR_GROUP_WORLD */
+    MPID_Group_free_hook(MPIR_Group_builtin + 2);       /* MPIR_GROUP_SELF  */
+#endif
+
     return MPI_SUCCESS;
 }
 
@@ -107,6 +119,9 @@ int MPIR_Group_release(MPIR_Group * group_ptr)
             /* Release session */
             MPIR_Session_release(group_ptr->session_ptr);
         }
+#ifdef MPID_DEV_GROUP_DECL
+        mpi_errno = MPID_Group_free_hook(group_ptr);
+#endif
         MPIR_Handle_obj_free(&MPIR_Group_mem, group_ptr);
     }
     return mpi_errno;
@@ -138,6 +153,9 @@ int MPIR_Group_create(int nproc, MPIR_Group ** new_group_ptr)
     (*new_group_ptr)->session_ptr = NULL;
     memset(&(*new_group_ptr)->pmap, 0, sizeof(struct MPIR_Pmap));
     (*new_group_ptr)->pmap.size = nproc;
+#ifdef MPID_DEV_GROUP_DECL
+    mpi_errno = MPID_Group_init_hook(*new_group_ptr);
+#endif
 
     return mpi_errno;
 }

src/mpid/ch3/include/mpidpre.h

@@ -195,6 +195,11 @@ typedef struct MPIDI_CH3I_comm
 }
 MPIDI_CH3I_comm_t;
 
+/* add vcrt to MPIR_Group so we can inherit it whenever possible */
+#define MPID_DEV_GROUP_DECL  struct MPIDI_VCRT *ch3_vcrt;
+int MPID_Group_init_hook(MPIR_Group * group_ptr);
+int MPID_Group_free_hook(MPIR_Group * group_ptr);
+
 #define MPID_DEV_COMM_DECL MPIDI_CH3I_comm_t dev;
 
 #ifndef DEFINED_REQ

src/mpid/ch3/src/ch3u_comm.c

@@ -581,3 +581,22 @@ void MPIDI_CH3I_Comm_find(int context_id, MPIR_Comm **comm)
 
     MPIR_FUNC_EXIT;
 }
+
+int MPID_Group_init_hook(MPIR_Group * group_ptr)
+{
+    group_ptr->ch3_vcrt = NULL;
+    return MPI_SUCCESS;
+}
+
+int MPID_Group_free_hook(MPIR_Group * group_ptr)
+{
+    int mpi_errno = MPI_SUCCESS;
+
+    if (group_ptr->ch3_vcrt) {
+        /* FIXME: setting TRUE so vc entries may get released.
+         *        Is there a case we don't want that?
+         */
+        mpi_errno = MPIDI_VCRT_Release(group_ptr->ch3_vcrt, TRUE);
+    }
+    return mpi_errno;
+}