Instrument to get some more details in python init phase.

examples/combozzle-mpi.py

@@ -96,8 +96,10 @@ def _get_box_mesh(dim, a, b, n, t=None, periodic=None):
     dim_names = ["x", "y", "z"]
     bttf = {}
     for i in range(dim):
-        bttf["-"+str(i+1)] = ["-"+dim_names[i]]
-        bttf["+"+str(i+1)] = ["+"+dim_names[i]]
+        if not periodic[i]:
+            bttf["-"+str(i+1)] = ["-"+dim_names[i]]
+            bttf["+"+str(i+1)] = ["+"+dim_names[i]]
+
     from meshmode.mesh.generation import generate_regular_rect_mesh as gen
     return gen(a=a, b=b, n=n, boundary_tag_to_face=bttf, mesh_type=t,
                periodic=periodic)
@@ -180,7 +182,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True,
 
     comm.Barrier()
     if rank == 0:
-        print(f"Main start: {time.ctime(time.time())}")
+        print(f"Hello: {time.ctime(time.time())}")
     comm.Barrier()
 
     from mirgecom.simutil import global_reduce as _global_reduce
@@ -225,7 +227,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True,
     nhealth = 100
     nrestart = 1000
     do_checkpoint = 0
-    boundary_report = 0
+    boundary_report = 1
     do_callbacks = 0
 
     # }}}  Time stepping control
@@ -597,7 +599,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True,
 
     comm.Barrier()
     if rank == 0:
-        print(f"ACTX setup start: {time.ctime(time.time())}")
+        print(f"Queue/ACTX setup start: {time.ctime(time.time())}")
     comm.Barrier()
 
     if use_profiling:
@@ -614,6 +616,11 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True,
     else:
         actx = actx_class(comm, queue, allocator=alloc, force_device_scalars=True)
 
+    comm.Barrier()
+    if rank == 0:
+        print(f"Queue/ACTX setup done: {time.ctime(time.time())}")
+    comm.Barrier()
+
     rst_path = "restart_data/"
     rst_pattern = (
         rst_path + "{cname}-{step:04d}-{rank:04d}.pkl"
@@ -638,6 +645,11 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True,
                                                                     generate_mesh)
         local_nelements = local_mesh.nelements
 
+    comm.Barrier()
+    if rank == 0:
+        print(f"Mesh setup done: {time.ctime(time.time())}")
+    comm.Barrier()
+
     print(f"{rank=},{dim=},{order=},{local_nelements=},{global_nelements=}")
     if grid_only:
         return 0
@@ -648,7 +660,7 @@ def main(ctx_factory=cl.create_some_context, use_logmgr=True,
 
     comm.Barrier()
     if rank == 0:
-        print(f"ACTX Setup end -> Solution init start: {time.ctime(time.time())}")
+        print(f"Discretization Setup end: {time.ctime(time.time())}")
     comm.Barrier()
 
     def _compiled_stepper_wrapper(state, t, dt, rhs):
@@ -733,6 +745,11 @@ def vol_max(x):
                 ("min_temperature", "------- T (min, max) (K)  = ({value:7g}, "),
                 ("max_temperature",    "{value:7g})\n")])
 
+    comm.Barrier()
+    if rank == 0:
+        print(f"Solution init start: {time.ctime(time.time())}")
+    comm.Barrier()
+
     if single_gas_only:
         nspecies = 0
         init_y = 0
@@ -959,6 +976,11 @@ def _sponge(cv):
                         f" {eq_pressure=}, {eq_temperature=},"
                         f" {eq_density=}, {eq_mass_fractions=}")
 
+    comm.Barrier()
+    if rank == 0:
+        print(f"Solution init done: {time.ctime(time.time())}")
+    comm.Barrier()
+
     def my_write_status(dt, cfl, dv=None):
         status_msg = f"------ {dt=}" if constant_cfl else f"----- {cfl=}"
         if ((dv is not None) and (not log_dependent)):
@@ -1241,6 +1263,10 @@ def dummy_rhs(t, state):
 
     comm.Barrier()
 
+    if rank == 0:
+        print(f"Simulation end time: {time.ctime(time.time())}")
+
+
     finish_tol = 1e-16
     assert np.abs(current_t - t_final) < finish_tol
 
@@ -1251,10 +1277,9 @@ def dummy_rhs(t, state):
             logger.info("Fluid solution failed health check.")
         raise MyRuntimeError("Failed simulation health check.")
 
-    if rank == 0:
-        print(f"Simulation end time: {time.ctime(time.time())}")
-
     comm.Barrier()
+    if rank == 0:
+        print(f"Goodbye: {time.ctime(time.time())}")
 
 
 if __name__ == "__main__":
@@ -1278,8 +1303,8 @@ def dummy_rhs(t, state):
     parser.add_argument("--restart_file", help="root name of restart file")
     parser.add_argument("--casename", help="casename to use for i/o")
     args = parser.parse_args()
-    from warnings import warn
-    warn("Automatically turning off DV logging. MIRGE-Com Issue(578)")
+    # from warnings import warn
+    # warn("Automatically turning off DV logging. MIRGE-Com Issue(578)")
     lazy = args.lazy
     log_dependent = False
     force_eval = not args.no_force
@@ -1304,7 +1329,7 @@ def dummy_rhs(t, state):
     else:
         print("No user input file, using default values")
 
-    print(f"Calling main: {time.ctime(time.time())}")
+    # print(f"Calling main: {time.ctime(time.time())}")
 
     main(use_logmgr=args.log, use_leap=args.leap, input_file=input_file,
          use_overintegration=args.overintegration,

mirgecom/simutil.py

@@ -457,6 +457,7 @@ def generate_and_distribute_mesh(comm, generate_mesh):
     global_nelements : :class:`int`
         The number of elements in the serial mesh
     """
+    import time
     from warnings import warn
     warn(
         "generate_and_distribute_mesh is deprecated and will go away Q4 2022. "
@@ -500,16 +501,19 @@ def distribute_mesh(comm, get_mesh_data, partition_generator_func=None):
     global_nelements: :class:`int`
         The number of elements in the global mesh
     """
+    import time
     from meshmode.distributed import mpi_distribute
 
     num_ranks = comm.Get_size()
+    rank = comm.Get_rank()
 
     if partition_generator_func is None:
         def partition_generator_func(mesh, tag_to_elements, num_ranks):
             from meshmode.distributed import get_partition_by_pymetis
             return get_partition_by_pymetis(mesh, num_ranks)
 
-    if comm.Get_rank() == 0:
+    if rank == 0:
+        print(f"Mesh generation begin: {time.ctime(time.time())}")
         global_data = get_mesh_data()
 
         from meshmode.mesh import Mesh
@@ -522,16 +526,21 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks):
         else:
             raise TypeError("Unexpected result from get_mesh_data")
 
+        print(f"Mesh generation end: {time.ctime(time.time())}")
+        print(f"Mesh partition begin: {time.ctime(time.time())}")
         from meshmode.mesh.processing import partition_mesh
 
         rank_per_element = partition_generator_func(mesh, tag_to_elements, num_ranks)
+        print(f"Mesh partition_generator_func done: {time.ctime(time.time())}")
 
         if tag_to_elements is None:
             rank_to_elements = {
                 rank: np.where(rank_per_element == rank)[0]
                 for rank in range(num_ranks)}
+            print(f"Mesh partition e-conn inverted: {time.ctime(time.time())}")
 
             rank_to_mesh_data = partition_mesh(mesh, rank_to_elements)
+            print(f"Mesh partition_mesh done: {time.ctime(time.time())}")
 
         else:
             tag_to_volume = {
@@ -561,6 +570,7 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks):
             part_id_to_part_index = {
                 part_id: part_index
                 for part_index, part_id in enumerate(part_id_to_elements.keys())}
+
             from meshmode.mesh.processing import _compute_global_elem_to_part_elem
             global_elem_to_part_elem = _compute_global_elem_to_part_elem(
                 mesh.nelements, part_id_to_elements, part_id_to_part_index,
@@ -589,6 +599,8 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks):
                     for vol in volumes}
                 for rank in range(num_ranks)}
 
+        print(f"Mesh partitioning done: {time.ctime(time.time())}")
+
         local_mesh_data = mpi_distribute(
             comm, source_rank=0, source_data=rank_to_mesh_data)
 
@@ -599,6 +611,9 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks):
 
         global_nelements = comm.bcast(None, root=0)
 
+    comm.Barrier()
+    if rank == 0:
+        print(f"Mesh distribution done: {time.ctime(time.time())}")
     return local_mesh_data, global_nelements