From 6426aae2e4d30501974409b6a8d8b9cf030198da Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Thu, 7 Sep 2023 06:10:32 -0700 Subject: [PATCH 01/55] Add contextmanager import --- mirgecom/simutil.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index eebe03316..d348228a6 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -74,6 +74,8 @@ import sys from functools import partial from typing import TYPE_CHECKING, Dict, List, Optional +from contextlib import contextmanager + from logpyle import IntervalTimer import grudge.op as op From 98a951125ab6874deab1551059dc552a79e90ceb Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 7 Sep 2023 08:23:04 -0500 Subject: [PATCH 02/55] Add mesh partitioning utilities --- mirgecom/simutil.py | 88 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index d348228a6..610ce757b 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -890,6 +890,94 @@ def generate_and_distribute_mesh(comm, generate_mesh, **kwargs): return distribute_mesh(comm, generate_mesh) +def _partition_single_volume_mesh( + mesh, num_ranks, rank_per_element, *, return_ranks=None): + rank_to_elements = { + rank: np.where(rank_per_element == rank)[0] + for rank in range(num_ranks)} + + from meshmode.mesh.processing import partition_mesh + return partition_mesh( + mesh, rank_to_elements, return_parts=return_ranks) + + +def _partition_multi_volume_mesh( + mesh, num_ranks, rank_per_element, tag_to_elements, volume_to_tags, *, + return_ranks=None): + if return_ranks is None: + return_ranks = [i for i in range(num_ranks)] + + tag_to_volume = { + tag: vol + for vol, tags in volume_to_tags.items() + for tag in tags} + + volumes = list(volume_to_tags.keys()) + + volume_index_per_element = np.full(mesh.nelements, -1, dtype=int) + for tag, elements in tag_to_elements.items(): + volume_index_per_element[elements] = volumes.index( + tag_to_volume[tag]) + + if np.any(volume_index_per_element < 0): + raise ValueError("Missing volume specification for some elements.") + + part_id_to_elements = { + PartID(volumes[vol_idx], rank): + np.where( + (volume_index_per_element == vol_idx) + & (rank_per_element == rank))[0] + for vol_idx in range(len(volumes)) + for rank in range(num_ranks)} + + # FIXME: Find a better way to do this + part_id_to_part_index = { + part_id: part_index + for part_index, part_id in enumerate(part_id_to_elements.keys())} + from meshmode.mesh.processing import _compute_global_elem_to_part_elem + global_elem_to_part_elem = _compute_global_elem_to_part_elem( + mesh.nelements, part_id_to_elements, part_id_to_part_index, + mesh.element_id_dtype) + + tag_to_global_to_part = { + tag: global_elem_to_part_elem[elements, :] + for tag, elements in tag_to_elements.items()} + + part_id_to_tag_to_elements = {} + for part_id in part_id_to_elements.keys(): + part_idx = part_id_to_part_index[part_id] + part_tag_to_elements = {} + for tag, global_to_part in tag_to_global_to_part.items(): + part_tag_to_elements[tag] = global_to_part[ + global_to_part[:, 0] == part_idx, 1] + part_id_to_tag_to_elements[part_id] = part_tag_to_elements + + return_parts = { + PartID(vol, rank) + for vol in volumes + for rank in return_ranks} + + from meshmode.mesh.processing import partition_mesh + part_id_to_mesh = partition_mesh( + mesh, part_id_to_elements, return_parts=return_parts) + + return { + rank: { + vol: ( + part_id_to_mesh[PartID(vol, rank)], + part_id_to_tag_to_elements[PartID(vol, rank)]) + for vol in volumes} + for rank in return_ranks} + + +@contextmanager +def _manage_mpi_comm(comm): + try: + yield comm + finally: + comm.Free() + + def distribute_mesh(comm, get_mesh_data, partition_generator_func=None, logmgr=None): r"""Distribute a mesh among all ranks in *comm*. From 1583d72553d98e4ff283ee0555b0c8281155659f Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 7 Sep 2023 09:26:45 -0500 Subject: [PATCH 03/55] Add majosm node-local mesh partitioning funcs --- mirgecom/simutil.py | 197 ++++++++++++++++++-------------------------- 1 file changed, 79 insertions(+), 118 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 610ce757b..0dee3006c 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -905,7 +905,7 @@ def _partition_multi_volume_mesh( mesh, num_ranks, rank_per_element, tag_to_elements, volume_to_tags, *, return_ranks=None): if return_ranks is None: - return_ranks = [i for i in range(num_ranks)] + return_ranks = list(range(num_ranks)) tag_to_volume = { tag: vol @@ -1014,10 +1014,12 @@ def distribute_mesh(comm, get_mesh_data, partition_generator_func=None, logmgr=N global_nelements: :class:`int` The number of elements in the global mesh """ + from mpi4py import MPI from mpi4py.util import pkl5 - comm_wrapper = pkl5.Intracomm(comm) + from meshmode.distributed import mpi_distribute + # pkl5_comm = pkl5.Intracomm(comm) - num_ranks = comm_wrapper.Get_size() + num_ranks = comm.Get_size() t_mesh_dist = IntervalTimer("t_mesh_dist", "Time spent distributing mesh data.") t_mesh_data = IntervalTimer("t_mesh_data", "Time spent getting mesh data.") t_mesh_part = IntervalTimer("t_mesh_part", "Time spent partitioning the mesh.") @@ -1028,132 +1030,91 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks): from meshmode.distributed import get_partition_by_pymetis return get_partition_by_pymetis(mesh, num_ranks) - if comm_wrapper.Get_rank() == 0: - if logmgr: - logmgr.add_quantity(t_mesh_data) - with t_mesh_data.get_sub_timer(): + with _manage_mpi_comm( + comm.Split_type(MPI.COMM_TYPE_SHARED, comm.Get_rank(), MPI.INFO_NULL) + ) as node_comm: + node_comm_wrapper = pkl5.Intracomm(node_comm) + node_ranks = node_comm_wrapper.gather(comm.Get_rank(), root=0) + my_node_rank = node_comm_wrapper.Get_rank() + + if my_node_rank == 0: + if logmgr: + logmgr.add_quantity(t_mesh_data) + with t_mesh_data.get_sub_timer(): + global_data = get_mesh_data() + else: global_data = get_mesh_data() - else: - global_data = get_mesh_data() - - from meshmode.mesh import Mesh - if isinstance(global_data, Mesh): - mesh = global_data - tag_to_elements = None - volume_to_tags = None - elif isinstance(global_data, tuple) and len(global_data) == 3: - mesh, tag_to_elements, volume_to_tags = global_data - else: - raise TypeError("Unexpected result from get_mesh_data") - if logmgr: - logmgr.add_quantity(t_mesh_part) - with t_mesh_part.get_sub_timer(): + from meshmode.mesh import Mesh + if isinstance(global_data, Mesh): + mesh = global_data + tag_to_elements = None + volume_to_tags = None + elif isinstance(global_data, tuple) and len(global_data) == 3: + mesh, tag_to_elements, volume_to_tags = global_data + else: + raise TypeError("Unexpected result from get_mesh_data") + + if logmgr: + logmgr.add_quantity(t_mesh_part) + with t_mesh_part.get_sub_timer(): + rank_per_element = \ + partition_generator_func(mesh, tag_to_elements, + num_ranks) + else: rank_per_element = partition_generator_func(mesh, tag_to_elements, num_ranks) - else: - rank_per_element = partition_generator_func(mesh, tag_to_elements, - num_ranks) - def get_rank_to_mesh_data(): - from meshmode.mesh.processing import partition_mesh - if tag_to_elements is None: - rank_to_elements = { - rank: np.where(rank_per_element == rank)[0] - for rank in range(num_ranks)} + def get_rank_to_mesh_data(): + if tag_to_elements is None: + rank_to_mesh_data = _partition_single_volume_mesh( + mesh, num_ranks, rank_per_element, + return_ranks=node_ranks) + else: + rank_to_mesh_data = _partition_multi_volume_mesh( + mesh, num_ranks, rank_per_element, tag_to_elements, + volume_to_tags, return_ranks=node_ranks) - rank_to_mesh_data_dict = partition_mesh(mesh, rank_to_elements) + rank_to_node_rank = { + rank: node_rank + for node_rank, rank in enumerate(node_ranks)} - rank_to_mesh_data = [ - rank_to_mesh_data_dict[rank] - for rank in range(num_ranks)] + node_rank_to_mesh_data = { + rank_to_node_rank[rank]: mesh_data + for rank, mesh_data in rank_to_mesh_data.items()} - else: - tag_to_volume = { - tag: vol - for vol, tags in volume_to_tags.items() - for tag in tags} - - volumes = list(volume_to_tags.keys()) - - volume_index_per_element = np.full(mesh.nelements, -1, dtype=int) - for tag, elements in tag_to_elements.items(): - volume_index_per_element[elements] = volumes.index( - tag_to_volume[tag]) - - if np.any(volume_index_per_element < 0): - raise ValueError("Missing volume specification " - "for some elements.") - - part_id_to_elements = { - PartID(volumes[vol_idx], rank): - np.where( - (volume_index_per_element == vol_idx) - & (rank_per_element == rank))[0] - for vol_idx in range(len(volumes)) - for rank in range(num_ranks)} - - # TODO: Add a public meshmode function to accomplish this? So we're - # not depending on meshmode internals - part_id_to_part_index = { - part_id: part_index - for part_index, part_id in enumerate(part_id_to_elements.keys())} - from meshmode.mesh.processing import \ - _compute_global_elem_to_part_elem - global_elem_to_part_elem = _compute_global_elem_to_part_elem( - mesh.nelements, part_id_to_elements, part_id_to_part_index, - mesh.element_id_dtype) - - tag_to_global_to_part = { - tag: global_elem_to_part_elem[elements, :] - for tag, elements in tag_to_elements.items()} - - part_id_to_tag_to_elements = {} - for part_id in part_id_to_elements.keys(): - part_idx = part_id_to_part_index[part_id] - part_tag_to_elements = {} - for tag, global_to_part in tag_to_global_to_part.items(): - part_tag_to_elements[tag] = global_to_part[ - global_to_part[:, 0] == part_idx, 1] - part_id_to_tag_to_elements[part_id] = part_tag_to_elements - - part_id_to_mesh = partition_mesh(mesh, part_id_to_elements) - - rank_to_mesh_data = [ - { - vol: ( - part_id_to_mesh[PartID(vol, rank)], - part_id_to_tag_to_elements[PartID(vol, rank)]) - for vol in volumes} - for rank in range(num_ranks)] - - return rank_to_mesh_data - - if logmgr: - logmgr.add_quantity(t_mesh_split) - with t_mesh_split.get_sub_timer(): - rank_to_mesh_data = get_rank_to_mesh_data() - else: - rank_to_mesh_data = get_rank_to_mesh_data() - - global_nelements = comm_wrapper.bcast(mesh.nelements, root=0) + return node_rank_to_mesh_data - if logmgr: - logmgr.add_quantity(t_mesh_dist) - with t_mesh_dist.get_sub_timer(): - local_mesh_data = comm_wrapper.scatter(rank_to_mesh_data, root=0) - else: - local_mesh_data = comm_wrapper.scatter(rank_to_mesh_data, root=0) + if logmgr: + logmgr.add_quantity(t_mesh_split) + with t_mesh_split.get_sub_timer(): + node_rank_to_mesh_data = get_rank_to_mesh_data() + else: + node_rank_to_mesh_data = get_rank_to_mesh_data() - else: - global_nelements = comm_wrapper.bcast(None, root=0) + global_nelements = node_comm_wrapper.bcast(mesh.nelements, root=0) - if logmgr: - logmgr.add_quantity(t_mesh_dist) - with t_mesh_dist.get_sub_timer(): - local_mesh_data = comm_wrapper.scatter(None, root=0) - else: - local_mesh_data = comm_wrapper.scatter(None, root=0) + if logmgr: + logmgr.add_quantity(t_mesh_dist) + with t_mesh_dist.get_sub_timer(): + local_mesh_data = mpi_distribute( + node_comm_wrapper, source_rank=0, + source_data=node_rank_to_mesh_data) + else: + local_mesh_data = mpi_distribute( + node_comm_wrapper, source_rank=0, + source_data=node_rank_to_mesh_data) + + else: # my_node_rank > 0, get mesh part from MPI + global_nelements = node_comm_wrapper.bcast(None, root=0) + + if logmgr: + logmgr.add_quantity(t_mesh_dist) + with t_mesh_dist.get_sub_timer(): + local_mesh_data = \ + mpi_distribute(node_comm_wrapper, source_rank=0) + else: + local_mesh_data = mpi_distribute(node_comm_wrapper, source_rank=0) return local_mesh_data, global_nelements From 550d485c1bbe7840e20c292d0ca3a9e48f97d14c Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Thu, 7 Sep 2023 12:12:21 -0700 Subject: [PATCH 04/55] Dispense with pkl5 comm wrapper which doesnt play nice with splitted comm. --- mirgecom/simutil.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 0dee3006c..d3d355cdc 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -1015,9 +1015,7 @@ def distribute_mesh(comm, get_mesh_data, partition_generator_func=None, logmgr=N The number of elements in the global mesh """ from mpi4py import MPI - from mpi4py.util import pkl5 from meshmode.distributed import mpi_distribute - # pkl5_comm = pkl5.Intracomm(comm) num_ranks = comm.Get_size() t_mesh_dist = IntervalTimer("t_mesh_dist", "Time spent distributing mesh data.") @@ -1033,9 +1031,8 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks): with _manage_mpi_comm( comm.Split_type(MPI.COMM_TYPE_SHARED, comm.Get_rank(), MPI.INFO_NULL) ) as node_comm: - node_comm_wrapper = pkl5.Intracomm(node_comm) - node_ranks = node_comm_wrapper.gather(comm.Get_rank(), root=0) - my_node_rank = node_comm_wrapper.Get_rank() + node_ranks = node_comm.gather(comm.Get_rank(), root=0) + my_node_rank = node_comm.Get_rank() if my_node_rank == 0: if logmgr: @@ -1092,29 +1089,29 @@ def get_rank_to_mesh_data(): else: node_rank_to_mesh_data = get_rank_to_mesh_data() - global_nelements = node_comm_wrapper.bcast(mesh.nelements, root=0) + global_nelements = node_comm.bcast(mesh.nelements, root=0) if logmgr: logmgr.add_quantity(t_mesh_dist) with t_mesh_dist.get_sub_timer(): local_mesh_data = mpi_distribute( - node_comm_wrapper, source_rank=0, + node_comm, source_rank=0, source_data=node_rank_to_mesh_data) else: local_mesh_data = mpi_distribute( - node_comm_wrapper, source_rank=0, + node_comm, source_rank=0, source_data=node_rank_to_mesh_data) else: # my_node_rank > 0, get mesh part from MPI - global_nelements = node_comm_wrapper.bcast(None, root=0) + global_nelements = node_comm.bcast(None, root=0) if logmgr: logmgr.add_quantity(t_mesh_dist) with t_mesh_dist.get_sub_timer(): local_mesh_data = \ - mpi_distribute(node_comm_wrapper, source_rank=0) + mpi_distribute(node_comm, source_rank=0) else: - local_mesh_data = mpi_distribute(node_comm_wrapper, source_rank=0) + local_mesh_data = mpi_distribute(node_comm, source_rank=0) return local_mesh_data, global_nelements From f09077f4f6fcb10763147d753b68c10c8aed040e Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 8 Sep 2023 08:51:56 -0500 Subject: [PATCH 05/55] Update mirgecom/simutil.py Co-authored-by: Matt Smith --- mirgecom/simutil.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index d3d355cdc..134d00ca2 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -930,7 +930,8 @@ def _partition_multi_volume_mesh( for vol_idx in range(len(volumes)) for rank in range(num_ranks)} - # FIXME: Find a better way to do this + # TODO: Add a public meshmode function to accomplish this? So we're + # not depending on meshmode internals part_id_to_part_index = { part_id: part_index for part_index, part_id in enumerate(part_id_to_elements.keys())} From 15a0eb98c9e2fe7308a2882b0dae174226267a76 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 8 Sep 2023 11:02:07 -0500 Subject: [PATCH 06/55] rearrange comming in mesh dist --- mirgecom/simutil.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 134d00ca2..b9e709f15 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -1019,6 +1019,8 @@ def distribute_mesh(comm, get_mesh_data, partition_generator_func=None, logmgr=N from meshmode.distributed import mpi_distribute num_ranks = comm.Get_size() + my_global_rank = comm.Get_rank() + t_mesh_dist = IntervalTimer("t_mesh_dist", "Time spent distributing mesh data.") t_mesh_data = IntervalTimer("t_mesh_data", "Time spent getting mesh data.") t_mesh_part = IntervalTimer("t_mesh_part", "Time spent partitioning the mesh.") @@ -1031,9 +1033,12 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks): with _manage_mpi_comm( comm.Split_type(MPI.COMM_TYPE_SHARED, comm.Get_rank(), MPI.INFO_NULL) - ) as node_comm: + ) as node_comm: node_ranks = node_comm.gather(comm.Get_rank(), root=0) my_node_rank = node_comm.Get_rank() + reader_color = 0 if my_node_rank == 0 else 1 + reader_comm = comm.Split(reader_color, my_global_rank) + my_reader_rank = reader_comm.Get_rank() if my_node_rank == 0: if logmgr: @@ -1043,6 +1048,10 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks): else: global_data = get_mesh_data() + reader_comm.Barrier() + if my_reader_rank == 0: + print("Mesh reading done on all nodes.") + from meshmode.mesh import Mesh if isinstance(global_data, Mesh): mesh = global_data @@ -1060,8 +1069,13 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks): partition_generator_func(mesh, tag_to_elements, num_ranks) else: - rank_per_element = partition_generator_func(mesh, tag_to_elements, - num_ranks) + rank_per_element = \ + partition_generator_func(mesh, tag_to_elements, + num_ranks) + + reader_comm.Barrier() + if my_reader_rank == 0: + print("Mesh partitioning done on each node.") def get_rank_to_mesh_data(): if tag_to_elements is None: @@ -1090,6 +1104,10 @@ def get_rank_to_mesh_data(): else: node_rank_to_mesh_data = get_rank_to_mesh_data() + reader_comm.Barrier() + if my_reader_rank == 0: + print("Rank-to-mesh data done on each node, distributing.") + global_nelements = node_comm.bcast(mesh.nelements, root=0) if logmgr: From 4f1eba22708df62384b0c68399c4aed860442c00 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Tue, 12 Sep 2023 19:05:56 -0700 Subject: [PATCH 07/55] Batch the reads --- mirgecom/logging_quantities.py | 7 +++++ mirgecom/simutil.py | 49 +++++++++++++++++++++++++++------- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/mirgecom/logging_quantities.py b/mirgecom/logging_quantities.py index c7a1064d3..6271ce620 100644 --- a/mirgecom/logging_quantities.py +++ b/mirgecom/logging_quantities.py @@ -32,6 +32,7 @@ .. autoclass:: DeviceMemoryUsage .. autofunction:: initialize_logmgr .. autofunction:: logmgr_add_cl_device_info +.. autofunction:: logmgr_add_simulation_info .. autofunction:: logmgr_add_device_memory_usage .. autofunction:: logmgr_add_many_discretization_quantities .. autofunction:: logmgr_add_mempool_usage @@ -101,6 +102,12 @@ def logmgr_add_cl_device_info(logmgr: LogManager, queue: cl.CommandQueue) -> Non logmgr.set_constant("cl_platform_version", dev.platform.version) +def logmgr_add_simulation_info(logmgr: LogManager, sim_info) -> None: + """Add some user-defined information to the logpyle output.""" + for field_name in sim_info: + logmgr.set_constant(field_name, sim_info[field_name]) + + def logmgr_add_device_name(logmgr: LogManager, queue: cl.CommandQueue): # noqa: D401 """Deprecated. Do not use in new code.""" from warnings import warn diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index b9e709f15..59f340520 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -979,7 +979,8 @@ def _manage_mpi_comm(comm): comm.Free() -def distribute_mesh(comm, get_mesh_data, partition_generator_func=None, logmgr=None): +def distribute_mesh(comm, get_mesh_data, partition_generator_func=None, logmgr=None, + num_per_batch=None): r"""Distribute a mesh among all ranks in *comm*. Retrieve the global mesh data with the user-supplied function *get_mesh_data*, @@ -1016,10 +1017,13 @@ def distribute_mesh(comm, get_mesh_data, partition_generator_func=None, logmgr=N The number of elements in the global mesh """ from mpi4py import MPI + from mpi4py.util import pkl5 + from socket import gethostname from meshmode.distributed import mpi_distribute num_ranks = comm.Get_size() my_global_rank = comm.Get_rank() + hostname = gethostname() t_mesh_dist = IntervalTimer("t_mesh_dist", "Time spent distributing mesh data.") t_mesh_data = IntervalTimer("t_mesh_data", "Time spent getting mesh data.") @@ -1032,8 +1036,10 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks): return get_partition_by_pymetis(mesh, num_ranks) with _manage_mpi_comm( - comm.Split_type(MPI.COMM_TYPE_SHARED, comm.Get_rank(), MPI.INFO_NULL) + pkl5.Intracomm(comm.Split_type(MPI.COMM_TYPE_SHARED, + comm.Get_rank(), MPI.INFO_NULL)) ) as node_comm: + node_ranks = node_comm.gather(comm.Get_rank(), root=0) my_node_rank = node_comm.Get_rank() reader_color = 0 if my_node_rank == 0 else 1 @@ -1041,12 +1047,33 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks): my_reader_rank = reader_comm.Get_rank() if my_node_rank == 0: + + num_reading_ranks = reader_comm.Get_size() + num_per_batch = num_per_batch or num_reading_ranks + num_reading_batches = max(int(num_reading_ranks / num_per_batch), 1) + read_batch = int(my_reader_rank / num_per_batch) + + print(f"Reading(rank, batch): ({my_reader_rank}, " + f"{read_batch}) on {hostname}.") + if logmgr: logmgr.add_quantity(t_mesh_data) with t_mesh_data.get_sub_timer(): - global_data = get_mesh_data() + for reading_batch in range(num_reading_batches): + if read_batch == reading_batch: + print(f"Reading mesh on {hostname}.") + global_data = get_mesh_data() + reader_comm.Barrier() + else: + reader_comm.Barrier() else: - global_data = get_mesh_data() + for reading_batch in range(num_reading_batches): + if read_batch == reading_batch: + print(f"Reading mesh on {hostname}.") + global_data = get_mesh_data() + reader_comm.Barrier() + else: + reader_comm.Barrier() reader_comm.Barrier() if my_reader_rank == 0: @@ -1062,6 +1089,10 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks): else: raise TypeError("Unexpected result from get_mesh_data") + reader_comm.Barrier() + if my_reader_rank == 0: + print("Making partition table on all nodes.") + if logmgr: logmgr.add_quantity(t_mesh_part) with t_mesh_part.get_sub_timer(): @@ -1073,10 +1104,6 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks): partition_generator_func(mesh, tag_to_elements, num_ranks) - reader_comm.Barrier() - if my_reader_rank == 0: - print("Mesh partitioning done on each node.") - def get_rank_to_mesh_data(): if tag_to_elements is None: rank_to_mesh_data = _partition_single_volume_mesh( @@ -1097,6 +1124,10 @@ def get_rank_to_mesh_data(): return node_rank_to_mesh_data + reader_comm.Barrier() + if my_reader_rank == 0: + print("Partitioning mesh on all nodes.") + if logmgr: logmgr.add_quantity(t_mesh_split) with t_mesh_split.get_sub_timer(): @@ -1106,7 +1137,7 @@ def get_rank_to_mesh_data(): reader_comm.Barrier() if my_reader_rank == 0: - print("Rank-to-mesh data done on each node, distributing.") + print("Partitioning done, distributing to node-local ranks.") global_nelements = node_comm.bcast(mesh.nelements, root=0) From 1fe0401eb1089caab51650e72c69b91030988b9a Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 12 Sep 2023 21:07:14 -0500 Subject: [PATCH 08/55] Add pkl dist method --- mirgecom/simutil.py | 207 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index eebe03316..53b396bb3 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -1068,6 +1068,213 @@ def get_rank_to_mesh_data(): return local_mesh_data, global_nelements +def distribute_mesh_pkl(comm, get_mesh_data, filename="mesh", + num_target_ranks=0, num_reader_ranks=0, + partition_generator_func=None, logmgr=None): + r"""Distribute a mesh among all ranks in *comm*. + + Retrieve the global mesh data with the user-supplied function *get_mesh_data*, + partition the mesh, and distribute it to every rank in the provided MPI + communicator *comm*. + + .. note:: + This is a collective routine and must be called by all MPI ranks. + + Parameters + ---------- + comm: + MPI communicator over which to partition the mesh + get_mesh_data: + Callable of zero arguments returning *mesh* or + *(mesh, tag_to_elements, volume_to_tags)*, where *mesh* is a + :class:`meshmode.mesh.Mesh`, *tag_to_elements* is a + :class:`dict` mapping mesh volume tags to :class:`numpy.ndarray`\ s of + element numbers, and *volume_to_tags* is a :class:`dict` that maps volumes + in the resulting distributed mesh to volume tags in *tag_to_elements*. + partition_generator_func: + Optional callable that takes *mesh*, *tag_to_elements*, and *comm*'s size, + and returns a :class:`numpy.ndarray` indicating to which rank each element + belongs. + + Returns + ------- + local_mesh_data: :class:`meshmode.mesh.Mesh` or :class:`dict` + If the result of calling *get_mesh_data* specifies a single volume, + *local_mesh_data* is the local mesh. If it specifies multiple volumes, + *local_mesh_data* will be a :class:`dict` mapping volume tags to + tuples of the form *(local_mesh, local_tag_to_elements)*. + global_nelements: :class:`int` + The number of elements in the global mesh + """ + from mpi4py.util import pkl5 + comm_wrapper = pkl5.Intracomm(comm) + + num_ranks = comm_wrapper.Get_size() + rank = comm_wrapper.Get_rank() + + if num_target_ranks <= 0: + num_target_ranks = num_ranks + if num_reader_ranks <= 0: + num_reader_ranks = num_ranks + + reader_color = 1 if rank < num_reader_ranks else 0 + reader_comm = comm_wrapper.Split(reader_color, rank) + reader_comm_wrapper = pkl5.Intracomm(reader_comm) + reader_rank = reader_comm_wrapper.Get_rank() + num_ranks_per_reader = int(num_target_ranks / num_reader_ranks) + num_leftover = num_target_ranks - (num_ranks_per_reader * num_reader_ranks) + num_ranks_this_reader = num_ranks_per_reader + (1 if reader_rank + < num_leftover else 0) + + t_mesh_dist = IntervalTimer("t_mesh_dist", "Time spent distributing mesh data.") + t_mesh_data = IntervalTimer("t_mesh_data", "Time spent getting mesh data.") + t_mesh_part = IntervalTimer("t_mesh_part", "Time spent partitioning the mesh.") + t_mesh_split = IntervalTimer("t_mesh_split", "Time spent splitting mesh parts.") + + if reader_color and num_ranks_this_reader > 0: + my_starting_rank = (num_ranks_per_reader * reader_rank + + reader_rank if reader_rank + < num_leftover else num_leftover) + my_ending_rank = my_starting_rank + num_ranks_this_reader - 1 + ranks_to_write = range(my_starting_rank, my_ending_rank) + print(f"{rank=}{reader_rank=}{my_starting_rank=}{my_ending_rank=}") + + if partition_generator_func is None: + def partition_generator_func(mesh, tag_to_elements, num_target_ranks): + from meshmode.distributed import get_partition_by_pymetis + return get_partition_by_pymetis(mesh, num_target_ranks) + + if logmgr: + logmgr.add_quantity(t_mesh_data) + with t_mesh_data.get_sub_timer(): + global_data = get_mesh_data() + else: + global_data = get_mesh_data() + + from meshmode.mesh import Mesh + if isinstance(global_data, Mesh): + mesh = global_data + tag_to_elements = None + volume_to_tags = None + elif isinstance(global_data, tuple) and len(global_data) == 3: + mesh, tag_to_elements, volume_to_tags = global_data + else: + raise TypeError("Unexpected result from get_mesh_data") + + if logmgr: + logmgr.add_quantity(t_mesh_part) + with t_mesh_part.get_sub_timer(): + rank_per_element = partition_generator_func(mesh, tag_to_elements, + num_target_ranks) + else: + rank_per_element = partition_generator_func(mesh, tag_to_elements, + num_target_ranks) + + def get_rank_to_mesh_data(): + from meshmode.mesh.processing import partition_mesh + if tag_to_elements is None: + rank_to_elements = { + rank: np.where(rank_per_element == rank)[0] + for rank in ranks_to_write} + + rank_to_mesh_data_dict = partition_mesh(mesh, rank_to_elements) + + rank_to_mesh_data = [ + rank_to_mesh_data_dict[rank] + for rank in ranks_to_write] + + else: + tag_to_volume = { + tag: vol + for vol, tags in volume_to_tags.items() + for tag in tags} + + volumes = list(volume_to_tags.keys()) + + volume_index_per_element = np.full(mesh.nelements, -1, dtype=int) + for tag, elements in tag_to_elements.items(): + volume_index_per_element[elements] = volumes.index( + tag_to_volume[tag]) + + if np.any(volume_index_per_element < 0): + raise ValueError("Missing volume specification " + "for some elements.") + + part_id_to_elements = { + PartID(volumes[vol_idx], rank): + np.where( + (volume_index_per_element == vol_idx) + & (rank_per_element == rank))[0] + for vol_idx in range(len(volumes)) + for rank in ranks_to_write} + + # TODO: Add a public meshmode function to accomplish this? So we're + # not depending on meshmode internals + part_id_to_part_index = { + part_id: part_index + for part_index, part_id in enumerate(part_id_to_elements.keys())} + from meshmode.mesh.processing import \ + _compute_global_elem_to_part_elem + global_elem_to_part_elem = _compute_global_elem_to_part_elem( + mesh.nelements, part_id_to_elements, part_id_to_part_index, + mesh.element_id_dtype) + + tag_to_global_to_part = { + tag: global_elem_to_part_elem[elements, :] + for tag, elements in tag_to_elements.items()} + + part_id_to_tag_to_elements = {} + for part_id in part_id_to_elements.keys(): + part_idx = part_id_to_part_index[part_id] + part_tag_to_elements = {} + for tag, global_to_part in tag_to_global_to_part.items(): + part_tag_to_elements[tag] = global_to_part[ + global_to_part[:, 0] == part_idx, 1] + part_id_to_tag_to_elements[part_id] = part_tag_to_elements + + part_id_to_mesh = partition_mesh(mesh, part_id_to_elements) + + rank_to_mesh_data = [ + { + vol: ( + part_id_to_mesh[PartID(vol, rank)], + part_id_to_tag_to_elements[PartID(vol, rank)]) + for vol in volumes} + for rank in ranks_to_write] + + return rank_to_mesh_data + + if logmgr: + logmgr.add_quantity(t_mesh_split) + with t_mesh_split.get_sub_timer(): + rank_to_mesh_data = get_rank_to_mesh_data(ranks_to_write) + else: + rank_to_mesh_data = get_rank_to_mesh_data(ranks_to_write) + + import os + import pickle + if logmgr: + logmgr.add_quantity(t_mesh_dist) + with t_mesh_dist.get_sub_timer(): + for rank in ranks_to_write: + rank_mesh_data = rank_to_mesh_data[rank] + mesh_data_to_pickle = (mesh.nelements, rank_mesh_data) + pkl_filename = filename + f"_rank{rank}.pkl" + if os.path.exists(pkl_filename): + os.remove(pkl_filename) + with open(pkl_filename, "wb") as pkl_file: + pickle.dump(mesh_data_to_pickle, pkl_file) + else: + for rank in ranks_to_write: + rank_mesh_data = rank_to_mesh_data[rank] + mesh_data_to_pickle = (mesh.nelements, rank_mesh_data) + pkl_filename = filename + f"_rank{rank}.pkl" + if os.path.exists(pkl_filename): + os.remove(pkl_filename) + with open(pkl_filename, "wb") as pkl_file: + pickle.dump(mesh_data_to_pickle, pkl_file) + + def extract_volumes(mesh, tag_to_elements, selected_tags, boundary_tag): r""" Create a mesh containing a subset of another mesh's volumes. From 46faa2d219bb6627a38b62e5928314403f2070b0 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Thu, 14 Sep 2023 08:36:46 -0700 Subject: [PATCH 09/55] Add mesh dist to pkl util --- mirgecom/simutil.py | 108 ++++++++++---------------------------------- 1 file changed, 24 insertions(+), 84 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index b456dbb5c..2822a6b8d 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -611,8 +611,9 @@ def geometric_mesh_partitioner(mesh, num_ranks=None, *, nranks_per_axis=None, # Create geometrically even partitions elem_to_rank = ((elem_centroids-x_min) / part_interval).astype(int) - - print(f"{elem_to_rank=}") + + if debug: + print(f"{elem_to_rank=}") # map partition id to list of elements in that partition part_to_elements = {r: set(np.where(elem_to_rank == r)[0]) @@ -970,7 +971,6 @@ def _partition_multi_volume_mesh( for vol in volumes} for rank in return_ranks} - @contextmanager def _manage_mpi_comm(comm): try: @@ -1208,15 +1208,15 @@ def distribute_mesh_pkl(comm, get_mesh_data, filename="mesh", comm_wrapper = pkl5.Intracomm(comm) num_ranks = comm_wrapper.Get_size() - rank = comm_wrapper.Get_rank() + my_rank = comm_wrapper.Get_rank() if num_target_ranks <= 0: num_target_ranks = num_ranks if num_reader_ranks <= 0: num_reader_ranks = num_ranks - reader_color = 1 if rank < num_reader_ranks else 0 - reader_comm = comm_wrapper.Split(reader_color, rank) + reader_color = 1 if my_rank < num_reader_ranks else 0 + reader_comm = comm_wrapper.Split(reader_color, my_rank) reader_comm_wrapper = pkl5.Intracomm(reader_comm) reader_rank = reader_comm_wrapper.Get_rank() num_ranks_per_reader = int(num_target_ranks / num_reader_ranks) @@ -1230,12 +1230,13 @@ def distribute_mesh_pkl(comm, get_mesh_data, filename="mesh", t_mesh_split = IntervalTimer("t_mesh_split", "Time spent splitting mesh parts.") if reader_color and num_ranks_this_reader > 0: - my_starting_rank = (num_ranks_per_reader * reader_rank - + reader_rank if reader_rank - < num_leftover else num_leftover) + my_starting_rank = num_ranks_per_reader * reader_rank + my_starting_rank = my_starting_rank + (reader_rank if reader_rank + < num_leftover else num_leftover) my_ending_rank = my_starting_rank + num_ranks_this_reader - 1 - ranks_to_write = range(my_starting_rank, my_ending_rank) - print(f"{rank=}{reader_rank=}{my_starting_rank=}{my_ending_rank=}") + ranks_to_write = list(range(my_starting_rank, my_ending_rank+1)) + + print(f"R({my_rank},{reader_rank}): W({my_starting_rank},{my_ending_rank})") if partition_generator_func is None: def partition_generator_func(mesh, tag_to_elements, num_target_ranks): @@ -1269,85 +1270,22 @@ def partition_generator_func(mesh, tag_to_elements, num_target_ranks): num_target_ranks) def get_rank_to_mesh_data(): - from meshmode.mesh.processing import partition_mesh if tag_to_elements is None: - rank_to_elements = { - rank: np.where(rank_per_element == rank)[0] - for rank in ranks_to_write} - - rank_to_mesh_data_dict = partition_mesh(mesh, rank_to_elements) - - rank_to_mesh_data = [ - rank_to_mesh_data_dict[rank] - for rank in ranks_to_write] - + rank_to_mesh_data = _partition_single_volume_mesh( + mesh, num_target_ranks, rank_per_element, + return_ranks=ranks_to_write) else: - tag_to_volume = { - tag: vol - for vol, tags in volume_to_tags.items() - for tag in tags} - - volumes = list(volume_to_tags.keys()) - - volume_index_per_element = np.full(mesh.nelements, -1, dtype=int) - for tag, elements in tag_to_elements.items(): - volume_index_per_element[elements] = volumes.index( - tag_to_volume[tag]) - - if np.any(volume_index_per_element < 0): - raise ValueError("Missing volume specification " - "for some elements.") - - part_id_to_elements = { - PartID(volumes[vol_idx], rank): - np.where( - (volume_index_per_element == vol_idx) - & (rank_per_element == rank))[0] - for vol_idx in range(len(volumes)) - for rank in ranks_to_write} - - # TODO: Add a public meshmode function to accomplish this? So we're - # not depending on meshmode internals - part_id_to_part_index = { - part_id: part_index - for part_index, part_id in enumerate(part_id_to_elements.keys())} - from meshmode.mesh.processing import \ - _compute_global_elem_to_part_elem - global_elem_to_part_elem = _compute_global_elem_to_part_elem( - mesh.nelements, part_id_to_elements, part_id_to_part_index, - mesh.element_id_dtype) - - tag_to_global_to_part = { - tag: global_elem_to_part_elem[elements, :] - for tag, elements in tag_to_elements.items()} - - part_id_to_tag_to_elements = {} - for part_id in part_id_to_elements.keys(): - part_idx = part_id_to_part_index[part_id] - part_tag_to_elements = {} - for tag, global_to_part in tag_to_global_to_part.items(): - part_tag_to_elements[tag] = global_to_part[ - global_to_part[:, 0] == part_idx, 1] - part_id_to_tag_to_elements[part_id] = part_tag_to_elements - - part_id_to_mesh = partition_mesh(mesh, part_id_to_elements) - - rank_to_mesh_data = [ - { - vol: ( - part_id_to_mesh[PartID(vol, rank)], - part_id_to_tag_to_elements[PartID(vol, rank)]) - for vol in volumes} - for rank in ranks_to_write] - + rank_to_mesh_data = _partition_multi_volume_mesh( + mesh, num_target_ranks, rank_per_element, tag_to_elements, + volume_to_tags, return_ranks=ranks_to_write) return rank_to_mesh_data if logmgr: logmgr.add_quantity(t_mesh_split) with t_mesh_split.get_sub_timer(): - rank_to_mesh_data = get_rank_to_mesh_data(ranks_to_write) + rank_to_mesh_data = get_rank_to_mesh_data() else: - rank_to_mesh_data = get_rank_to_mesh_data(ranks_to_write) + rank_to_mesh_data = get_rank_to_mesh_data() import os import pickle @@ -1355,7 +1293,8 @@ def get_rank_to_mesh_data(): logmgr.add_quantity(t_mesh_dist) with t_mesh_dist.get_sub_timer(): for rank in ranks_to_write: - rank_mesh_data = rank_to_mesh_data[rank] + local_rank_index = rank - my_starting_rank + rank_mesh_data = rank_to_mesh_data[local_rank_index] mesh_data_to_pickle = (mesh.nelements, rank_mesh_data) pkl_filename = filename + f"_rank{rank}.pkl" if os.path.exists(pkl_filename): @@ -1364,7 +1303,8 @@ def get_rank_to_mesh_data(): pickle.dump(mesh_data_to_pickle, pkl_file) else: for rank in ranks_to_write: - rank_mesh_data = rank_to_mesh_data[rank] + local_rank_index = rank - my_starting_rank + rank_mesh_data = rank_to_mesh_data[local_rank_index] mesh_data_to_pickle = (mesh.nelements, rank_mesh_data) pkl_filename = filename + f"_rank{rank}.pkl" if os.path.exists(pkl_filename): From f004abf8c53349ac0d6161ec680bcb143f47e4cb Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Fri, 15 Sep 2023 09:02:00 -0700 Subject: [PATCH 10/55] Add meshdist util --- bin/meshdist.py | 260 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100755 bin/meshdist.py diff --git a/bin/meshdist.py b/bin/meshdist.py new file mode 100755 index 000000000..68f0617d5 --- /dev/null +++ b/bin/meshdist.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python +"""mirgecom mesh distribution utility""" + +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" +import logging +import argparse +import sys +import os + +from pytools.obj_array import make_obj_array +from functools import partial + +from logpyle import IntervalTimer, set_dt +from mirgecom.logging_quantities import ( + initialize_logmgr, + logmgr_add_cl_device_info, + logmgr_set_time, + logmgr_add_simulation_info, + logmgr_add_device_memory_usage, + logmgr_add_mempool_usage, +) + +from mirgecom.simutil import ( + ApplicationOptionsError, + distribute_mesh, + distribute_mesh_pkl +) +from mirgecom.mpi import mpi_entry_point + + +class SingleLevelFilter(logging.Filter): + def __init__(self, passlevel, reject): + self.passlevel = passlevel + self.reject = reject + + def filter(self, record): + if self.reject: + return (record.levelno != self.passlevel) + else: + return (record.levelno == self.passlevel) + + +class MyRuntimeError(RuntimeError): + """Simple exception to kill the simulation.""" + + pass + + +@mpi_entry_point +def main(actx_class, mesh_source=None, ndist=None, + output_path=None, log_path=None, + casename=None, use_1d_part=None, use_wall=False): + + if mesh_source is None: + raise ApplicationOptionsError("Missing mesh source file.") + + mesh_source.strip("'") + + if log_path is None: + log_path = "log_data" + + log_path.strip("'") + + if output_path is None: + output_path = "." + output_path.strip("'") + + # control log messages + logger = logging.getLogger(__name__) + logger.propagate = False + + if (logger.hasHandlers()): + logger.handlers.clear() + + # send info level messages to stdout + h1 = logging.StreamHandler(sys.stdout) + f1 = SingleLevelFilter(logging.INFO, False) + h1.addFilter(f1) + logger.addHandler(h1) + + # send everything else to stderr + h2 = logging.StreamHandler(sys.stderr) + f2 = SingleLevelFilter(logging.INFO, True) + h2.addFilter(f2) + logger.addHandler(h2) + + from mpi4py import MPI + from mpi4py.util import pkl5 + comm_world = MPI.COMM_WORLD + comm = pkl5.Intracomm(comm_world) + rank = comm.Get_rank() + nparts = comm.Get_size() + + if ndist is None: + ndist = nparts + + if casename is None: + casename = f"mirgecom_np{nparts}" + casename.strip("'") + + if rank == 0: + print(f"Running mesh distribution on {nparts} MPI ranks.") + print(f"Casename: {casename}") + print(f"Mesh source file: {mesh_source}") + + # logging and profiling + logname = log_path + "/" + casename + ".sqlite" + + if rank == 0: + log_dir = os.path.dirname(logname) + if log_dir and not os.path.exists(log_dir): + os.makedirs(log_dir) + + comm.Barrier() + + logmgr = initialize_logmgr(True, + filename=logname, mode="wu", mpi_comm=comm) + + from mirgecom.array_context import initialize_actx, actx_class_is_profiling + actx = initialize_actx(actx_class, comm) + queue = getattr(actx, "queue", None) + use_profiling = actx_class_is_profiling(actx_class) + alloc = getattr(actx, "allocator", None) + + monitor_memory = True + monitor_performance = 2 + + logmgr_add_cl_device_info(logmgr, queue) + + if monitor_memory: + logmgr_add_device_memory_usage(logmgr, queue) + logmgr_add_mempool_usage(logmgr, alloc) + + logmgr.add_watches([ + ("memory_usage_python.max", + "| Memory:\n| \t python memory: {value:7g} Mb\n") + ]) + + try: + logmgr.add_watches([ + ("memory_usage_gpu.max", + "| \t gpu memory: {value:7g} Mb\n") + ]) + except KeyError: + pass + + logmgr.add_watches([ + ("memory_usage_hwm.max", + "| \t memory hwm: {value:7g} Mb\n")]) + + if rank == 0: + print(f"Reading mesh from {mesh_source}.") + print(f"Writing {ndist} mesh pkl files to {output_path}.") + + def get_mesh_data(): + from meshmode.mesh.io import read_gmsh + mesh, tag_to_elements = read_gmsh( + mesh_source, + return_tag_to_elements_map=True) + volume_to_tags = { + "fluid": ["fluid"]} + if use_wall: + volume_to_tags["wall"] = ["wall_insert", "wall_surround"] + else: + from mirgecom.simutil import extract_volumes + mesh, tag_to_elements = extract_volumes( + mesh, tag_to_elements, volume_to_tags["fluid"], + "wall_interface") + return mesh, tag_to_elements, volume_to_tags + + def my_partitioner(mesh, tag_to_elements, num_ranks): + from mirgecom.simutil import geometric_mesh_partitioner + return geometric_mesh_partitioner( + mesh, num_ranks, auto_balance=True, debug=False) + + part_func = my_partitioner if use_1d_part else None + + if os.path.exists(output_path): + if not os.path.isdir(output_path): + raise ApplicationOptionsError( + "Mesh dist mode requires \"output\"" + " parameter to be a directory for output.") + if rank == 0: + if not os.path.exists(output_path): + os.makedirs(output_path) + + comm.Barrier() + mesh_filename = output_path + "/" + casename + "_mesh" + + if rank == 0: + print(f"Writing mesh pkl files to {mesh_filename}.") + + distribute_mesh_pkl( + comm, get_mesh_data, filename=mesh_filename, + num_target_ranks=ndist, + partition_generator_func=part_func, logmgr=logmgr) + + comm.Barrier() + + logmgr_set_time(logmgr, 0, 0) + logmgr.tick_before() + logmgr.tick_after() + logmgr.close() + + +if __name__ == "__main__": + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + level=logging.INFO) + + parser = argparse.ArgumentParser( + description="MIRGE-Com Mesh Distribution") + parser.add_argument("-w", "--wall", dest="use_wall", + action="store_true", help="Include wall domain in mesh.") + parser.add_argument("-1", "--1dpart", dest="one_d_part", + action="store_true", help="Use 1D partitioner.") + parser.add_argument("-n", "--ndist", type=int, dest="ndist", + nargs="?", action="store", help="Number of distributed parts") + parser.add_argument("-s", "--source", type=str, dest="source", + nargs="?", action="store", help="Gmsh mesh source file") + parser.add_argument("-o", "--ouput-path", type=str, dest="output_path", + nargs="?", action="store", help="Output path for distributed mesh pkl files") + parser.add_argument("-c", "--casename", type=str, dest="casename", nargs="?", + action="store", help="Root name of distributed mesh pkl files.") + parser.add_argument("-g", "--logpath", type=str, dest="log_path", nargs="?", + action="store", help="simulation case name") + + args = parser.parse_args() + + from mirgecom.array_context import get_reasonable_array_context_class + actx_class = get_reasonable_array_context_class( + lazy=False, distributed=True, profiling=False, numpy=False) + + main(actx_class, mesh_source=args.source, + output_path=args.output_path, ndist=args.ndist, + log_path=args.log_path, casename=args.casename, + use_1d_part=args.one_d_part, use_wall=args.use_wall) From 372132615524318faa9905c4da7f29ca8bae118f Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Sat, 16 Sep 2023 14:59:09 -0700 Subject: [PATCH 11/55] Import at top, use better filenames, loop over actual items in parted meshes. --- bin/meshdist.py | 6 ++++-- mirgecom/simutil.py | 25 +++++++++++-------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/bin/meshdist.py b/bin/meshdist.py index 68f0617d5..48bef66e9 100755 --- a/bin/meshdist.py +++ b/bin/meshdist.py @@ -117,11 +117,11 @@ def main(actx_class, mesh_source=None, ndist=None, ndist = nparts if casename is None: - casename = f"mirgecom_np{nparts}" + casename = f"mirgecom_np{ndist}" casename.strip("'") if rank == 0: - print(f"Running mesh distribution on {nparts} MPI ranks.") + print(f"Distributing on {nparts} ranks into {ndist} parts.") print(f"Casename: {casename}") print(f"Mesh source file: {mesh_source}") @@ -220,7 +220,9 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): comm.Barrier() logmgr_set_time(logmgr, 0, 0) + logmgr logmgr.tick_before() + set_dt(logmgr, 0.) logmgr.tick_after() logmgr.close() diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 2822a6b8d..db10eca17 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -72,6 +72,8 @@ """ import logging import sys +import os +import pickle from functools import partial from typing import TYPE_CHECKING, Dict, List, Optional from contextlib import contextmanager @@ -1053,7 +1055,7 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks): num_reading_batches = max(int(num_reading_ranks / num_per_batch), 1) read_batch = int(my_reader_rank / num_per_batch) - print(f"Reading(rank, batch): ({my_reader_rank}, " + print(f"Read(rank, batch): Dist({my_reader_rank}, " f"{read_batch}) on {hostname}.") if logmgr: @@ -1236,7 +1238,8 @@ def distribute_mesh_pkl(comm, get_mesh_data, filename="mesh", my_ending_rank = my_starting_rank + num_ranks_this_reader - 1 ranks_to_write = list(range(my_starting_rank, my_ending_rank+1)) - print(f"R({my_rank},{reader_rank}): W({my_starting_rank},{my_ending_rank})") + print(f"R({my_rank},{reader_rank}): " + f"W({my_starting_rank},{my_ending_rank})") if partition_generator_func is None: def partition_generator_func(mesh, tag_to_elements, num_target_ranks): @@ -1287,26 +1290,20 @@ def get_rank_to_mesh_data(): else: rank_to_mesh_data = get_rank_to_mesh_data() - import os - import pickle if logmgr: logmgr.add_quantity(t_mesh_dist) with t_mesh_dist.get_sub_timer(): - for rank in ranks_to_write: - local_rank_index = rank - my_starting_rank - rank_mesh_data = rank_to_mesh_data[local_rank_index] - mesh_data_to_pickle = (mesh.nelements, rank_mesh_data) - pkl_filename = filename + f"_rank{rank}.pkl" + for part_rank, part_mesh in rank_to_mesh_data.items(): + pkl_filename = filename + f"_rank{part_rank}.pkl" + mesh_data_to_pickle = (mesh.nelements, part_mesh) if os.path.exists(pkl_filename): os.remove(pkl_filename) with open(pkl_filename, "wb") as pkl_file: pickle.dump(mesh_data_to_pickle, pkl_file) else: - for rank in ranks_to_write: - local_rank_index = rank - my_starting_rank - rank_mesh_data = rank_to_mesh_data[local_rank_index] - mesh_data_to_pickle = (mesh.nelements, rank_mesh_data) - pkl_filename = filename + f"_rank{rank}.pkl" + for part_rank, part_mesh in rank_to_mesh_data.items(): + pkl_filename = filename + f"_rank{part_rank}.pkl" + mesh_data_to_pickle = (mesh.nelements, part_mesh) if os.path.exists(pkl_filename): os.remove(pkl_filename) with open(pkl_filename, "wb") as pkl_file: From f9babfd4022a9e1181db5a503ebc8137d0b9c22e Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 21 Sep 2023 19:56:25 -0500 Subject: [PATCH 12/55] Add some utils to support m-to-n restart. --- mirgecom/simutil.py | 97 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 90 insertions(+), 7 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index db10eca17..e229a2f9a 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -613,7 +613,7 @@ def geometric_mesh_partitioner(mesh, num_ranks=None, *, nranks_per_axis=None, # Create geometrically even partitions elem_to_rank = ((elem_centroids-x_min) / part_interval).astype(int) - + if debug: print(f"{elem_to_rank=}") @@ -973,6 +973,7 @@ def _partition_multi_volume_mesh( for vol in volumes} for rank in return_ranks} + @contextmanager def _manage_mpi_comm(comm): try: @@ -1049,7 +1050,6 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks): my_reader_rank = reader_comm.Get_rank() if my_node_rank == 0: - num_reading_ranks = reader_comm.Get_size() num_per_batch = num_per_batch or num_reading_ranks num_reading_batches = max(int(num_reading_ranks / num_per_batch), 1) @@ -1272,6 +1272,14 @@ def partition_generator_func(mesh, tag_to_elements, num_target_ranks): rank_per_element = partition_generator_func(mesh, tag_to_elements, num_target_ranks) + # Save this little puppy for later (m-to-n restart support) + if reader_rank == 0: + part_table_fname = filename + f"_decomp_np{num_target_ranks}.pkl" + if os.path.exists(part_table_fname): + os.remove(part_table_fname) + with open(part_table_fname, "wb") as pkl_file: + pickle.dump(rank_per_element, pkl_file) + def get_rank_to_mesh_data(): if tag_to_elements is None: rank_to_mesh_data = _partition_single_volume_mesh( @@ -1294,7 +1302,7 @@ def get_rank_to_mesh_data(): logmgr.add_quantity(t_mesh_dist) with t_mesh_dist.get_sub_timer(): for part_rank, part_mesh in rank_to_mesh_data.items(): - pkl_filename = filename + f"_rank{part_rank}.pkl" + pkl_filename = filename + f"_rank{part_rank}.pkl" mesh_data_to_pickle = (mesh.nelements, part_mesh) if os.path.exists(pkl_filename): os.remove(pkl_filename) @@ -1302,7 +1310,7 @@ def get_rank_to_mesh_data(): pickle.dump(mesh_data_to_pickle, pkl_file) else: for part_rank, part_mesh in rank_to_mesh_data.items(): - pkl_filename = filename + f"_rank{part_rank}.pkl" + pkl_filename = filename + f"_rank{part_rank}.pkl" mesh_data_to_pickle = (mesh.nelements, part_mesh) if os.path.exists(pkl_filename): os.remove(pkl_filename) @@ -1379,9 +1387,84 @@ def extract_volumes(mesh, tag_to_elements, selected_tags, boundary_tag): return in_mesh, tag_to_in_elements +def interpart_mapping(target_decomp, source_decomp): + """Return a mapping of which partitions to source for the target decomp.""" + from collections import defaultdict + + interpart_map = defaultdict(set) + + for elemid, part in enumerate(target_decomp): + interpart_map[part].add(source_decomp[elemid]) + + for part in interpart_map: + interpart_map[part] = list(interpart_map[part]) + + return interpart_map + + +def global_element_list_for_decomp(decomp_map): + """Return a list of global elements for each partition.""" + from collections import defaultdict + global_elements_per_part = defaultdict(list) + + for elemid, part in enumerate(decomp_map): + global_elements_per_part[part].append(elemid) + + return global_elements_per_part + + +# Need a function to determine which of my local elements overlap +# with a disparate decomp part. Optionally restrict attention to +# selected parts. +# For each (or selected) new part +# Make a dict/map to hold element mapping (key: old_part, val: dict[elid]->[elid]) +# For each local element in the new part +# 1. find the global el id +# 2. grab the old part for that global el +# 3. find the old part-local id (i.e. index) for that global el +# 4. Append local and remote el id lists for old-part-specific dict entry +def interdecomposition_overlap(target_decomp_map, source_decomp_map, + return_parts=None): + """Map element indices for overlapping, disparate decompositions. + + For each (or optionally selected) target parts, this routine + returns a dictionary keyed by overlapping remote partitions from + the source decomposition, the value of which is a map from the + target-part-specific local indexes to the source-part-specific local + index of for the corresponding element. + + { targ_part_1 : { src_part_1 : { local_el_index : remote_el_index, ... }, + src_part_2 : { local_el_index : remote_el_index, ... }, + ... + }, + targ_part_2 : { ... }, + ... + } + + This data structure is useful for mapping the solution data from + the old decomp pkl restart files to the new decomp solution arrays. + """ + src_part_to_els = global_element_list_for_decomp(source_decomp_map) + trg_part_to_els = global_element_list_for_decomp(target_decomp_map) + ntrg_parts = len(trg_part_to_els) + if return_parts is None: + return_parts = list(range[ntrg_parts]) + overlap_maps = {} + for trg_part in return_parts: + part_el_map = {{}} + trg_els = trg_part_to_els[trg_part] + for glb_el in trg_els: + src_part = source_decomp_map[glb_el] + src_el_index = src_part_to_els[src_part].index(glb_el) + loc_el_index = trg_els.index(glb_el) + part_el_map[src_part][loc_el_index] = src_el_index + overlap_maps[trg_part] = part_el_map + return overlap_maps + + def boundary_report(dcoll, boundaries, outfile_name, *, dd=DD_VOLUME_ALL, mesh=None): - """Generate a report of the grid boundaries.""" + """Generate a report of the mesh boundaries.""" boundaries = normalize_boundaries(boundaries) comm = dcoll.mpi_communicator @@ -1729,7 +1812,7 @@ def get_topology(self): connectivity = a if connectivity is None: - raise ValueError("File is missing grid connectivity data") + raise ValueError("File is missing mesh connectivity data") return connectivity @@ -1741,7 +1824,7 @@ def get_geometry(self): geometry = a if geometry is None: - raise ValueError("File is missing grid node location data") + raise ValueError("File is missing mesh node location data") return geometry From c815fb719c64c83e88c658dde892938d7a0c2c53 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Mon, 25 Sep 2023 08:56:46 -0500 Subject: [PATCH 13/55] Add some utils to support m-to-n restart --- mirgecom/restart.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index d5b95bdc2..8d8c13981 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -54,3 +54,47 @@ def write_restart_file(actx, restart_data, filename, comm=None): with array_context_for_pickling(actx): with open(filename, "wb") as f: pickle.dump(restart_data, f) + + +def redist_restart_data(actx, src_data=None, rst_filename=None, mesh_filename=None, + get_mesh_data=None, partition_generator_func=None, + comm=None, logmgr=None): + """Redistribute the restart data.""" + # rank = 0 + nparts = 1 + nparts_old = src_data["num_parts"] + if comm: + nparts = comm.Get_size() + # rank = comm.Get_rank() + comm.Broadcast + if nparts == nparts_old: + return src_data + + # temporarily + return src_data + + +def perform_restart(actx, restart_filename, mesh_filename=None, + get_mesh_data=None, partition_generator_func=None, + comm=None, logmgr=None): + """Restart solution even if decomp changes.""" + import os + nparts = 1 + + if comm: + nparts = comm.Get_size() + + rst_data = {} + if os.path.exists(restart_filename): + with array_context_for_pickling(actx): + with open(restart_filename, "rb") as f: + rst_data = pickle.load(f) + if rst_data["num_parts"] == nparts: + return rst_data + + return redist_restart_data(actx, src_data=rst_data, + rst_filename=restart_filename, comm=comm, + mesh_filename=mesh_filename, + get_mesh_data=get_mesh_data, + partition_generator_func=partition_generator_func, + logmgr=logmgr) From 2b662d6b4ff3180b00b53d2b6604246b08498b5c Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 29 Sep 2023 00:04:05 -0500 Subject: [PATCH 14/55] Add stand-alone redist util. --- bin/meshdist.py | 15 +- bin/redist.py | 365 ++++++++++++++++++++++++ mirgecom/restart.py | 174 ++++++++--- mirgecom/simutil.py | 56 +++- test/data/M24k_mesh_decomp_np1_pkl_data | Bin 0 -> 163560 bytes test/data/M24k_mesh_decomp_np2_pkl_data | Bin 0 -> 163560 bytes test/data/M24k_mesh_decomp_np4_pkl_data | Bin 0 -> 163560 bytes test/test_restart.py | 128 ++++++++- 8 files changed, 674 insertions(+), 64 deletions(-) create mode 100644 bin/redist.py create mode 100644 test/data/M24k_mesh_decomp_np1_pkl_data create mode 100644 test/data/M24k_mesh_decomp_np2_pkl_data create mode 100644 test/data/M24k_mesh_decomp_np4_pkl_data diff --git a/bin/meshdist.py b/bin/meshdist.py index 48bef66e9..9fe61b10d 100755 --- a/bin/meshdist.py +++ b/bin/meshdist.py @@ -1,5 +1,4 @@ -#!/usr/bin/env python -"""mirgecom mesh distribution utility""" +"""Read gmsh mesh, partition it, and create a pkl file per mesh partition.""" __copyright__ = """ Copyright (C) 2020 University of Illinois Board of Trustees @@ -72,7 +71,7 @@ class MyRuntimeError(RuntimeError): def main(actx_class, mesh_source=None, ndist=None, output_path=None, log_path=None, casename=None, use_1d_part=None, use_wall=False): - + """The main function.""" if mesh_source is None: raise ApplicationOptionsError("Missing mesh source file.") @@ -200,7 +199,7 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): if os.path.exists(output_path): if not os.path.isdir(output_path): raise ApplicationOptionsError( - "Mesh dist mode requires \"output\"" + "Mesh dist mode requires 'output'" " parameter to be a directory for output.") if rank == 0: if not os.path.exists(output_path): @@ -218,7 +217,7 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): partition_generator_func=part_func, logmgr=logmgr) comm.Barrier() - + logmgr_set_time(logmgr, 0, 0) logmgr logmgr.tick_before() @@ -240,11 +239,13 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): parser.add_argument("-1", "--1dpart", dest="one_d_part", action="store_true", help="Use 1D partitioner.") parser.add_argument("-n", "--ndist", type=int, dest="ndist", - nargs="?", action="store", help="Number of distributed parts") + nargs="?", action="store", + help="Number of distributed parts") parser.add_argument("-s", "--source", type=str, dest="source", nargs="?", action="store", help="Gmsh mesh source file") parser.add_argument("-o", "--ouput-path", type=str, dest="output_path", - nargs="?", action="store", help="Output path for distributed mesh pkl files") + nargs="?", action="store", + help="Output path for distributed mesh pkl files") parser.add_argument("-c", "--casename", type=str, dest="casename", nargs="?", action="store", help="Root name of distributed mesh pkl files.") parser.add_argument("-g", "--logpath", type=str, dest="log_path", nargs="?", diff --git a/bin/redist.py b/bin/redist.py new file mode 100644 index 000000000..25ca771a8 --- /dev/null +++ b/bin/redist.py @@ -0,0 +1,365 @@ +"""Re-distribute a mirgecom restart dump and create a new restart dataset.""" + +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" +import logging +import argparse +import sys +import os +import glob +import pickle + +# from pytools.obj_array import make_obj_array +# from functools import partial + +from logpyle import ( + # IntervalTimer, + set_dt +) +from mirgecom.logging_quantities import ( + initialize_logmgr, + logmgr_add_cl_device_info, + logmgr_set_time, + logmgr_add_device_memory_usage, + logmgr_add_mempool_usage, +) + +from mirgecom.simutil import ( + ApplicationOptionsError, + distribute_mesh_pkl +) +from mirgecom.mpi import mpi_entry_point + + +class SingleLevelFilter(logging.Filter): + """Filter the logger.""" + + def __init__(self, passlevel, reject): + """Initialize the filter.""" + self.passlevel = passlevel + self.reject = reject + + def filter(self, record): + """Execute the filter.""" + if self.reject: + return (record.levelno != self.passlevel) + else: + return (record.levelno == self.passlevel) + + +class MyRuntimeError(RuntimeError): + """Simple exception to kill the simulation.""" + + pass + + +@mpi_entry_point +def main(actx_class, mesh_source=None, ndist=None, mdist=None, + output_path=None, input_path=None, log_path=None, + casename=None, use_1d_part=None, use_wall=False, + restart_file=None): + """Redistribute a mirgecom restart dataset.""" + if mesh_source is None: + raise ApplicationOptionsError("Missing mesh source file.") + + mesh_source.strip("'") + + if log_path is None: + log_path = "log_data" + + log_path.strip("'") + + if output_path is None: + output_path = "." + output_path.strip("'") + + if input_path is None: + raise ApplicationOptionsError("Input path/filename is required.") + + # control log messages + logger = logging.getLogger(__name__) + logger.propagate = False + + if (logger.hasHandlers()): + logger.handlers.clear() + + # send info level messages to stdout + h1 = logging.StreamHandler(sys.stdout) + f1 = SingleLevelFilter(logging.INFO, False) + h1.addFilter(f1) + logger.addHandler(h1) + + # send everything else to stderr + h2 = logging.StreamHandler(sys.stderr) + f2 = SingleLevelFilter(logging.INFO, True) + h2.addFilter(f2) + logger.addHandler(h2) + + from mpi4py import MPI + from mpi4py.util import pkl5 + comm_world = MPI.COMM_WORLD + comm = pkl5.Intracomm(comm_world) + rank = comm.Get_rank() + nprocs = comm.Get_size() + + # Default to decomp for one part per process + if ndist is None: + ndist = nprocs + + if mdist is None: + # Try to detect it. If can't then fail. + if rank == 0: + files = glob.glob(input_path) + xps = ["_decomp_", "_mesh_"] + ffiles = [f for f in files if not any(xc in f for xc in xps)] + mdist = len(ffiles) + if mdist <= 0: + ffiles = [f for f in files if "_decomp_" not in f] + mdist = len(ffiles) + if mdist <= 0: + mdist = len(files) + mdist = comm.bcast(mdist, root=0) + if mdist <= 0: + raise ApplicationOptionsError("Cannot detect number of parts " + "for input data.") + else: + if rank == 0: + print(f"Automatically detected {mdist} input parts.") + + # We need a decomp map for the input data + # If can't find, then generate one. + input_data_directory = os.path.dirname(input_path) + output_filename = os.path.basename(input_path) + casename = casename or output_filename + casename.strip("'") + + if os.path.exists(output_path): + if not os.path.isdir(output_path): + raise ApplicationOptionsError( + "Mesh dist mode requires 'output'" + " parameter to be a directory for output.") + if rank == 0: + if not os.path.exists(output_path): + os.makedirs(output_path) + + output_directory = output_path + output_path = output_directory + "/" + output_filename + mesh_filename = output_directory + "/" + casename + "_mesh" + + if output_path == input_data_directory: + raise ApplicationOptionsError("Output path must be different than input" + " location because of filename collisions.") + + decomp_map_file_search_pattern = \ + input_data_directory + f"/*_decomp_np{mdist}.pkl" + input_decomp_map_files = glob.glob(decomp_map_file_search_pattern) + source_decomp_map_file = \ + input_decomp_map_files[0] if input_decomp_map_files else None + + generate_source_decomp = \ + True if source_decomp_map_file is None else False + if source_decomp_map_file is None: + source_decomp_map_file = input_path + f"_decomp_np{mdist}.pkl" + if generate_source_decomp: + print("Unable to find source decomp map, generating from scratch.") + else: + print("Found existing source decomp map.") + print(f"Source decomp map file: {source_decomp_map_file}.") + + if rank == 0: + print(f"Redist on {nprocs} procs: {mdist}->{ndist} parts") + print(f"Casename: {casename}") + print(f"Mesh source file: {mesh_source}") + + # logging and profiling + logname = log_path + "/" + casename + ".sqlite" + + if rank == 0: + log_dir = os.path.dirname(logname) + if log_dir and not os.path.exists(log_dir): + os.makedirs(log_dir) + + comm.Barrier() + + logmgr = initialize_logmgr(True, + filename=logname, mode="wu", mpi_comm=comm) + + from mirgecom.array_context import initialize_actx + actx = initialize_actx(actx_class, comm) + queue = getattr(actx, "queue", None) + alloc = getattr(actx, "allocator", None) + + monitor_memory = True + + logmgr_add_cl_device_info(logmgr, queue) + + if monitor_memory: + logmgr_add_device_memory_usage(logmgr, queue) + logmgr_add_mempool_usage(logmgr, alloc) + + logmgr.add_watches([ + ("memory_usage_python.max", + "| Memory:\n| \t python memory: {value:7g} Mb\n") + ]) + + try: + logmgr.add_watches([ + ("memory_usage_gpu.max", + "| \t gpu memory: {value:7g} Mb\n") + ]) + except KeyError: + pass + + logmgr.add_watches([ + ("memory_usage_hwm.max", + "| \t memory hwm: {value:7g} Mb\n")]) + + if rank == 0: + print(f"Reading mesh from {mesh_source}.") + print(f"Writing {ndist} mesh pkl files to {output_path}.") + + def get_mesh_data(): + from meshmode.mesh.io import read_gmsh + mesh, tag_to_elements = read_gmsh( + mesh_source, + return_tag_to_elements_map=True) + volume_to_tags = { + "fluid": ["fluid"]} + if use_wall: + volume_to_tags["wall"] = ["wall_insert", "wall_surround"] + else: + from mirgecom.simutil import extract_volumes + mesh, tag_to_elements = extract_volumes( + mesh, tag_to_elements, volume_to_tags["fluid"], + "wall_interface") + return mesh, tag_to_elements, volume_to_tags + + def my_partitioner(mesh, tag_to_elements, num_ranks): + if use_1d_part: + from mirgecom.simutil import geometric_mesh_partitioner + return geometric_mesh_partitioner( + mesh, num_ranks, auto_balance=True, debug=False) + else: + from meshmode.distributed import get_partition_by_pymetis + return get_partition_by_pymetis(mesh, num_ranks) + + part_func = my_partitioner + + # This bit will write the source decomp's (M) partition table if it + # didn't already exist. We need this table to create the + # N-parted restart data from the M-parted data. + if generate_source_decomp: + if rank == 0: + print("Generating source decomp...") + source_mesh_data = get_mesh_data() + from meshmode.mesh import Mesh + if isinstance(source_mesh_data, Mesh): + source_mesh = source_mesh_data + tag_to_elements = None + volume_to_tags = None + elif isinstance(source_mesh_data, tuple): + source_mesh, tag_to_elements, volume_to_tags = source_mesh_data + else: + raise TypeError("Unexpected result from get_mesh_data") + rank_per_element = my_partitioner(source_mesh, tag_to_elements, mdist) + with open(source_decomp_map_file, "wb") as pkl_file: + pickle.dump(rank_per_element, pkl_file) + print("Done generating source decomp.") + + comm.Barrier() + + if rank == 0: + print(f"Partitioning mesh to {ndist} parts, writing to {mesh_filename}...") + + # This bit creates the N-parted mesh pkl files and partition table + distribute_mesh_pkl( + comm, get_mesh_data, filename=mesh_filename, + num_target_ranks=ndist, partition_generator_func=part_func, logmgr=logmgr) + + comm.Barrier() + + if rank == 0: + print("Done partitioning target mesh, mesh pkl files written.") + print(f"Generating the restart data for {ndist} parts...") + + target_decomp_map_file = mesh_filename + f"_decomp_np{ndist}.pkl" + from mirgecom.simutil import redistribute_restart_data + redistribute_restart_data(actx, comm, source_decomp_map_file, input_path, + target_decomp_map_file, output_path, mesh_filename) + + if rank == 0: + print("Done generating restart data.") + print(f"Restart data for {ndist} parts is in {output_path}") + + logmgr_set_time(logmgr, 0, 0) + logmgr + logmgr.tick_before() + set_dt(logmgr, 0.) + logmgr.tick_after() + logmgr.close() + + +if __name__ == "__main__": + + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + level=logging.INFO) + + parser = argparse.ArgumentParser( + description="MIRGE-Com Mesh Distribution") + parser.add_argument("-w", "--wall", dest="use_wall", + action="store_true", help="Include wall domain in mesh.") + parser.add_argument("-1", "--1dpart", dest="one_d_part", + action="store_true", help="Use 1D partitioner.") + parser.add_argument("-n", "--ndist", type=int, dest="ndist", + nargs="?", action="store", + help="Number of distributed parts") + parser.add_argument("-m", "--mdist", type=int, dest="mdist", + nargs="?", action="store", + help="Number of source data parts") + parser.add_argument("-s", "--source", type=str, dest="source", + nargs="?", action="store", help="Gmsh mesh source file") + parser.add_argument("-o", "--ouput-path", type=str, dest="output_path", + nargs="?", action="store", + help="Output directory for distributed mesh pkl files") + parser.add_argument("-i", "--input-path", type=str, dest="input_path", + nargs="?", action="store", + help="Input path/root filename for restart pkl files") + parser.add_argument("-c", "--casename", type=str, dest="casename", nargs="?", + action="store", + help="Root name of distributed mesh pkl files.") + parser.add_argument("-g", "--logpath", type=str, dest="log_path", nargs="?", + action="store", help="simulation case name") + + args = parser.parse_args() + + from mirgecom.array_context import get_reasonable_array_context_class + actx_class = get_reasonable_array_context_class( + lazy=False, distributed=True, profiling=False, numpy=False) + + main(actx_class, mesh_source=args.source, + output_path=args.output_path, ndist=args.ndist, + input_path=args.input_path, mdist=args.mdist, + log_path=args.log_path, casename=args.casename, + use_1d_part=args.one_d_part, use_wall=args.use_wall) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index 8d8c13981..3db380152 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -56,45 +56,135 @@ def write_restart_file(actx, restart_data, filename, comm=None): pickle.dump(restart_data, f) -def redist_restart_data(actx, src_data=None, rst_filename=None, mesh_filename=None, - get_mesh_data=None, partition_generator_func=None, - comm=None, logmgr=None): - """Redistribute the restart data.""" - # rank = 0 - nparts = 1 - nparts_old = src_data["num_parts"] - if comm: - nparts = comm.Get_size() - # rank = comm.Get_rank() - comm.Broadcast - if nparts == nparts_old: - return src_data - - # temporarily - return src_data - - -def perform_restart(actx, restart_filename, mesh_filename=None, - get_mesh_data=None, partition_generator_func=None, - comm=None, logmgr=None): - """Restart solution even if decomp changes.""" - import os - nparts = 1 - - if comm: - nparts = comm.Get_size() - - rst_data = {} - if os.path.exists(restart_filename): - with array_context_for_pickling(actx): - with open(restart_filename, "rb") as f: - rst_data = pickle.load(f) - if rst_data["num_parts"] == nparts: - return rst_data - - return redist_restart_data(actx, src_data=rst_data, - rst_filename=restart_filename, comm=comm, - mesh_filename=mesh_filename, - get_mesh_data=get_mesh_data, - partition_generator_func=partition_generator_func, - logmgr=logmgr) +def redistribute_restart_data(actx, comm, source_decomp_map_file, input_path, + target_decomp_map_file, output_path, + mesh_filename): + """Redistribute a restart dataset M-to-N.""" + from meshmode.dof_array import DOFArray + from dataclasses import is_dataclass, asdict + from mirgecom.simutil import ( + invert_decomp, + interdecomposition_overlap, + copy_mapped_dof_array_data + ) + from mpi4py.util import pkl5 + + def _create_zeros_like_dofarray(actx, nel, ary): + # Get nnodes from the shape of the sample DOFArray + _, nnodes = ary[0].shape + zeros_array = actx.zeros((nel, nnodes)) + return DOFArray(actx, (zeros_array,)) + + def _recursive_map_and_copy(trg_item, src_item, elem_map): + """Recursively map and copy DOFArrays from src_item.""" + if isinstance(src_item, DOFArray): + if trg_item is None: + raise ValueError("No corresponding target DOFArray found.") + return copy_mapped_dof_array_data(trg_item, src_item, elem_map) + elif isinstance(src_item, dict): + return {k: _recursive_map_and_copy(trg_item.get(k, None), v, elem_map) + for k, v in src_item.items()} + elif isinstance(src_item, (list, tuple)): + return type(src_item)(_recursive_map_and_copy(t, v, elem_map) + for t, v in zip(trg_item, src_item)) + elif is_dataclass(src_item): + trg_dict = asdict(trg_item) + return type(src_item)(**{ + k: _recursive_map_and_copy(trg_dict.get(k, None), v, + elem_map) + for k, v in asdict(src_item).items()}) + else: + return src_item + + def _recursive_init_with_zeros(sample_item, trg_zeros): + """Recursively initialize data structures with zeros or original data.""" + black_list = ["volume_to_local_mesh_data", "mesh"] + if isinstance(sample_item, DOFArray): + return 1. * trg_zeros + elif isinstance(sample_item, dict): + return {k: _recursive_init_with_zeros(v, trg_zeros) + for k, v in sample_item.items() if k not in black_list} + elif isinstance(sample_item, (list, tuple)): + return type(sample_item)(_recursive_init_with_zeros(v, trg_zeros) + for v in sample_item) + elif is_dataclass(sample_item): + return type(sample_item)(**{k: _recursive_init_with_zeros(v, + trg_zeros) + for k, v in asdict(sample_item).items()}) + else: + return sample_item + + comm_wrapper = pkl5.Intracomm(comm) + my_rank = comm_wrapper.Get_rank() + + with open(source_decomp_map_file, "rb") as pkl_file: + src_dcmp = pickle.load(pkl_file) + with open(target_decomp_map_file, "rb") as pkl_file: + trg_dcmp = pickle.load(pkl_file) + + trg_parts = invert_decomp(trg_dcmp) + trg_nparts = len(trg_parts) + + writer_color = 1 if my_rank < trg_nparts else 0 + writer_comm = comm_wrapper.Split(writer_color, my_rank) + writer_comm_wrapper = pkl5.Intracomm(writer_comm) + + if writer_color: + writer_nprocs = writer_comm_wrapper.Get_size() + writer_rank = writer_comm_wrapper.Get_rank() + nparts_per_writer = int(writer_nprocs / trg_nparts) + nleftover = trg_nparts - (nparts_per_writer * writer_nprocs) + nparts_this_writer = nparts_per_writer + (1 if writer_rank + < nleftover else 0) + my_starting_rank = nparts_per_writer * writer_rank + my_starting_rank = my_starting_rank + (writer_rank if writer_rank + < nleftover else nleftover) + my_ending_rank = my_starting_rank + nparts_this_writer - 1 + parts_to_write = list(range(my_starting_rank, my_ending_rank+1)) + xdo = interdecomposition_overlap(trg_dcmp, src_dcmp, + return_parts=parts_to_write) + + # Read one source restart file to get the 'order' and a sample DOFArray + mesh_data_item = "volume_to_local_mesh_data" + sample_restart_file = f"{input_path}={writer_rank:04d}.pkl" + with open(sample_restart_file, "rb") as f: + sample_data = pickle.load(f) + if "mesh" in sample_data: + mesh_data_item = "mesh" # check mesh data return type instead + sample_dof_array = \ + next(val for key, val in + sample_data.items() if isinstance(val, DOFArray)) + + for trg_part, olaps in xdo.items(): + # Number of elements in each target partition + trg_nel = len(trg_parts[trg_part]) + + # Create representative DOFArray with zeros using sample for order + # But with nelem = trg_nel + trg_zeros = _create_zeros_like_dofarray( + actx, trg_nel, sample_dof_array) + # Use trg_zeros to reset all dof arrays in the out_rst_data to + # the current (trg_part) size made of zeros. + out_rst_data = _recursive_init_with_zeros(sample_data, trg_zeros) + + # Read and Map DOFArrays from source to target + for src_part, elem_map in olaps.items(): + # elem_map={trg-local-index : src-local-index} for trg,src parts + src_restart_file = f"{input_path}-{src_part:04d}.pkl" + with open(src_restart_file, "rb") as f: + src_rst_data = pickle.load(f) + out_rst_data = _recursive_map_and_copy( + out_rst_data, src_rst_data, elem_map) + + # Read new mesh data and stack it in the restart file + mesh_pkl_filename = f"{mesh_filename}_np{trg_nparts}_rank{trg_part}.pkl" + with open(mesh_pkl_filename, "rb") as pkl_file: + global_nelements, volume_to_local_mesh_data = pickle.load(pkl_file) + out_rst_data[mesh_data_item] = volume_to_local_mesh_data + + # Write out the trg_part-specific redistributed pkl restart file + output_filename = f"{output_path}-{trg_part:04d}.pkl" + with open(output_filename, "wb") as f: + pickle.dump(out_rst_data, f) + + return diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index e229a2f9a..daf309641 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -1302,7 +1302,8 @@ def get_rank_to_mesh_data(): logmgr.add_quantity(t_mesh_dist) with t_mesh_dist.get_sub_timer(): for part_rank, part_mesh in rank_to_mesh_data.items(): - pkl_filename = filename + f"_rank{part_rank}.pkl" + pkl_filename = (filename + + f"_np{num_target_ranks}_rank{part_rank}.pkl") mesh_data_to_pickle = (mesh.nelements, part_mesh) if os.path.exists(pkl_filename): os.remove(pkl_filename) @@ -1387,22 +1388,45 @@ def extract_volumes(mesh, tag_to_elements, selected_tags, boundary_tag): return in_mesh, tag_to_in_elements -def interpart_mapping(target_decomp, source_decomp): +def copy_mapped_dof_array_data(trg_dof_array, src_dof_array, index_map): + """Copy data between DOFArrays from disparate meshes.""" + # Assume ONE group (tetrahedra ONLY) + actx = trg_dof_array.array_context + trg_dof_array_np = actx.to_numpy(trg_dof_array) + src_dof_array_np = actx.to_numpy(src_dof_array) + + trg_array = trg_dof_array_np[0] + src_array = src_dof_array_np[0] + src_nel, src_nnodes = src_array.shape + trg_nel, trg_nnodes = trg_array.shape + + if src_nnodes != trg_nnodes: + raise ValueError("DOFArray mapped copy must be of same order.") + + # Actual data copy + for trg_el, src_el in index_map.items(): + trg_array[trg_el] = src_array[src_el] + + # trg_dof_array_np[0] = trg_array + return actx.from_numpy(trg_dof_array_np) + + +def interdecomposition_mapping(target_decomp, source_decomp): """Return a mapping of which partitions to source for the target decomp.""" from collections import defaultdict - interpart_map = defaultdict(set) + interdecomp_map = defaultdict(set) for elemid, part in enumerate(target_decomp): - interpart_map[part].add(source_decomp[elemid]) + interdecomp_map[part].add(source_decomp[elemid]) - for part in interpart_map: - interpart_map[part] = list(interpart_map[part]) + for part in interdecomp_map: + interdecomp_map[part] = list(interdecomp_map[part]) - return interpart_map + return interdecomp_map -def global_element_list_for_decomp(decomp_map): +def invert_decomp(decomp_map): """Return a list of global elements for each partition.""" from collections import defaultdict global_elements_per_part = defaultdict(list) @@ -1444,21 +1468,25 @@ def interdecomposition_overlap(target_decomp_map, source_decomp_map, This data structure is useful for mapping the solution data from the old decomp pkl restart files to the new decomp solution arrays. """ - src_part_to_els = global_element_list_for_decomp(source_decomp_map) - trg_part_to_els = global_element_list_for_decomp(target_decomp_map) + src_part_to_els = invert_decomp(source_decomp_map) + trg_part_to_els = invert_decomp(target_decomp_map) + ipmap = interdecomposition_mapping(target_decomp_map, source_decomp_map) + ntrg_parts = len(trg_part_to_els) if return_parts is None: - return_parts = list(range[ntrg_parts]) + return_parts = list(range(ntrg_parts)) overlap_maps = {} for trg_part in return_parts: - part_el_map = {{}} + overlap_maps[trg_part] = {} + for src_part in ipmap[trg_part]: + overlap_maps[trg_part][src_part] = {} + for trg_part in return_parts: trg_els = trg_part_to_els[trg_part] for glb_el in trg_els: src_part = source_decomp_map[glb_el] src_el_index = src_part_to_els[src_part].index(glb_el) loc_el_index = trg_els.index(glb_el) - part_el_map[src_part][loc_el_index] = src_el_index - overlap_maps[trg_part] = part_el_map + overlap_maps[trg_part][src_part][loc_el_index] = src_el_index return overlap_maps diff --git a/test/data/M24k_mesh_decomp_np1_pkl_data b/test/data/M24k_mesh_decomp_np1_pkl_data new file mode 100644 index 0000000000000000000000000000000000000000..2f11c044c815852778f5b2f5c3b167666dc2a4f2 GIT binary patch literal 163560 zcmeIuJ4yp#5CGt8Vz9CG4pylxth5ORSMUOo$ASU#VEt315fOq0{w|Y4c`6^-9KrIN zVy2h}!}oTwe{VIB&*!Vza@`v*>Z&(eJ*Vlgu7~T$XODF?Ud&%oy&9+3MeZI3BcIME z&B!hWtxen%qxgu2xEK_7U-z4M>g4Vut(Vo`PJ4P8+s~%m$;DNaWqJGk8;3M#r5J6m z`>*ZeGYSC$1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+z<&$uyT^->V$}Nq D?o>Vg literal 0 HcmV?d00001 diff --git a/test/data/M24k_mesh_decomp_np2_pkl_data b/test/data/M24k_mesh_decomp_np2_pkl_data new file mode 100644 index 0000000000000000000000000000000000000000..5c44854b8f64a489dba2f5e22b74ce1919100140 GIT binary patch literal 163560 zcmeI3&CYIDafEG%#U#_*Lq;BFkWrYRSU3_lfNWzaNccyGdVZpI$zC?{_ah zegD&!-@p9g<)1%#|L1@HkKeug=a1ig@T<4qe*O9X&;01u|MK$3KRnt$e)s;rzI^=g z$N&D9|NHB&|MAg#Z(lzB? zCAYfc@kO#Zw0^%w`*^7S8spUdq($;&iCe3bC2Zjl7s$El)HCib#8uEebyaW zpHUt2;H(^0`RJX>SM9}@eIWHL_ReMf(Xq!NI)}{BS5Ez|IQu=3U(_eZtPXt^Bj2c> z-ILYz?u+K4{_Oi^?aSVUu61v#KHAgkhi@aM(iTcjBwSLte?$Uj55Z!CV)*M>(uQk_ad-v?` zKHE?8UhS{@l<(fJ_dFY)mG3Mcm+xwSxA(OADLHTlcv?=U+*A2H5;*01Ty;l%_t*K| zuljwlC;FX5eSBu4+w1CA?Vr8pD!(YVsDIU+>~Z(FI&;0a%D4BP)pN7nJIiPFd&~dv zr7DN-=MepVqyA`~>aTp%KRdsA?=1gWKG|Hdx?OkaUgdg!)K~p%-nze4&%M2K%ojcq z)EuH*UEkH!|5Tp6KdaMsUOlJ#TY2{0tgdxG`m8@Xka^J6>1R=WlxKY+-6~(V_o{u_ zJz1T5Pj7wK-rjp>&+&c`f4Cz3S%`nHAM;sckJCqe=FmkL>9r46WRKRfoO#gA(Qifk zbdI`1)hl;%uCDW~9AD<(5k`9YyO8^%{aIbs$N5or82M{|mXG>6?)Fi4bJ;pRpqrzg zMd#z{>g#k-U44{C^V-*WxYKDJ&KyV`q^?Ncq3W|7zeta+74c;b?hqZ=s$*^xb+7u- z*E&4vqf=xK)P9z`dFqPJ7oF>C9X+UhFBjeu|^M z%#ZrIdFt$m5IxwdXI>FM>Tt!VuX;FhirR;R=;2vp4$RIuUF1Vw5x=O8E{ph~hbz(t zU0>^2j&CpOo?aiFQ(t&ihaO~3IdzbJgymu4$*^7M;)C*bP+P2)uDG7`9ytuBVB7h@{Rhv`|wq4 z^>ccBqrU2tyLngF`H1)WFsJ_1!Kja}71dWce$=zb+-lT&TK6#@&7sR8eopWD)Ipth zb#&T?gQ}w*p_`-Li@H~Rv>$QiXXzrJ)_!zf)Mp>M2+=Dtr}bWrpTl0?yUgjkXWxzg zsP8NvcSqJA9f%(s)Vgw~qwWx&QC#icdf#dHa6fm!L06}rMSMn4_u;2?~`( z)n)bgJ5(Qd7S-3ysZP0@qwe%tKZU!yxlj9WMf4zjcouce@u;r-Xn$58>9ntW)*X=# zefB#@r8q0hl8r4uBbZ4T^&FAAoWpf^;5m_(R;G<>~)Bb z_Mcc(M73qWY;k}skK?mx-sP6js z&$lb;zo?(xhY#2~m({oG z^d9BvN52&#AMHD?d!0_}%u$axx-2?BtvkQ0&dpPIKGa(gUyymY!>rEDYhC>u&(@tk z^B_9qTBi?cAFdeb>4VI}TXA&0b>D0~_k&sg)%xrmr|L8x<*j#*&bRJct$()ntj??a zuT9|8-{{MLtpgb$^9=eI-x{R05U2DGAK_ z@2gopn#(F*-E#_WeQ&G2_nmH!)|L0}b92t0I_T!3x|?_PUcYFLe(zr96e|uXEB8v^tOV-M z!}q^O1N!0gs%hx6|;HhoN|1q!xiaA$eik2-RUAdeGnhz z)D_Wz+MmU<{-e5Ro_>V98=d3SLG&PX#i&23*LlbFj)-S{A{~1anWGK|U7db}v*y(Y z4n}v3_H|DyAKkxdo_B!kQBFNV=0MfEy6R5lr{C}Hqps+3?$znsX z(fL#E$iCz1d!v2y72SK#_4eHy^~fJxVJ?BnGndn;s``G{ewo>M;|FG6)2h$<-bJ_j z{cgXjYp&=(`tS(RgXkUCI(jhbqnnNF1@TwjTW7BDk?8Ck+;^J2r`PZF`+DDXx4HTb zYySM*?ROB}yX$wddhcrgDA(%e{`|FmC-fjW!xiaYh3vm-Uv^L3^++I_*Ijh8zH6&b ztNt!|>Yc0Kss76K9_6k2+4HS_qxb5*EWc{+sy*tfJexySx7t5@M^?A{-rf7NIqlZ< z=A?7VuX@+6y}j?t-ka66-ZARy=3Tw@zS2LEe3l$e`CZJqW0ucdzwfH|kNRfkv%22z zv)aG+uHAFHa@n*`>wsM-*NQL(Rsahwx9a%=BK@n zdlYxvf#XZPOa&g^^8&3@PEe!M%2v)_~5J6kvF9iu*@^Rshl_0#*6xBBhYb3dr} zxVq||g`fJ~Q}0^+-SyqdSMSgI*IkbUTwa<_#GTJ*UH6XqW#`%JP<_y8{jU7f_ndmy zUBC0LyH|h5)&9G3b9?o@%H5pSd*{?=HJ^3oET7fyRsY?0uin4fU-QiJQ}1&3>%Gd| zTx)%mZ|fbq^zQwxKI_g=|K9oDxmoX??UQ{^RyX^;@_Qt|D~I~-d-LD*-D_{|yN~|) zU>s_nf6Z!I9bjqn`QRkIseInhcug)uv_PhQpALp<2 z)%(<6x!&_EeD^zd?{EE%ce}UseWibV;mYBMa~S<|p!3Q#|ERC}+1%}S^=NZ`tMw9oZjebt?J@vU4Q3GDh_tMi}b-|N#mr{80ikNOsWBzt-e zqu+mY-sQ8(*X^tNn%`^m9i#s2?-K2c`m^s@|9po_Efzsfy(=b^QwG)IG^gaB{2GX(s|`=*ZjR&{gmrO)JKmFqz<||eTU3}s&jSK z!CUprIh^h1_N=b&dIx*K-gl19_wIFb*?Y1&-J^5LRS$PMS7$Clomb9$YdxCF_R&Qc z-A5mEeo^1eMRoOcoH~dvJd3?^S^ukayaRN5>4Q#3-RWFCnxlUeMtABR=t0-l`Y4a~ zX&(+o`<>3!@yqtnfjUneq_3!bxWi~(`*8f=4$*<=lv4+_4+m989n`+^Rvr4)=hw=Ry;C`NYF#-x(Dhwi z=OP{YR#e}JJAdXBBYkThf9BvI^;zhAwXR(EyS}UIJiJxUoZ>8>t9*6Ov-9ltK?iCs zu8uD1qw7WdLFc1=IOyi6XLaZu;scKA+zSA@3 zkU9GBRvevg-KTop4+mA>%d>vyW+Qhy->C2AsE^L$6X8{RqB~COpPtL<_s#yEukF3F z^IPS|cLCX>JgU>bJGyt@{atsvzrMpG0sU^XbD8a@_nyvs-`_iTdLG&PM&Ca=pS^Rn z?pfZkI;UCw5MRSAN=iSKX6+cUHIR&b#?umDg2!R_FgL?^vA^ z{#WId-4p4wubg)kK9ZT8LpJZMZuWhr`DqTyPrJA7d3>SP9PajA-M!R7&B4{tMSXP7 zj+MhBfzj`LmmKTP-1ojqPNTnDoqz4`zWV*6`$p&a{-Zvl^W5QZwf|{*xo6kA*?&4_ z^NDm(e^n0Iy;)uFovZzO@2a`SmpLVe?C&J2yXsxN_w>%SzAM@j^>yEBz3zjz?qzNi zb#KIVpX#+={E-kjJWF2s4&}VJ7iar%$86u(e%$#icuGE}-0>{m)xBd^&b@n?1Kqn; z)!C~!YahO|{O;!S+TVZb_geK`)K__H9;5TE`>xjSdf%?SSLeHH-|T#6`|Zl*u6wV_ ze{|32{HpKMd)Ml@UjNxSopQ(QclG-1&V&76?>p``7rl41KP!*u4&BqL*ZEbPyTDa@ z%J-3Q<&eu^_3tt3pVjU9u6mzxcUM;D=3RZ2uiKZc&$=t}iTc@lSLtTmH_L~2&i2E% z@R2MzXwJ&fIc(KCpS#vu-;W-=+qrap72UgC{jTp^eRpeKt@_pXJO8`f`7HNq-pbuOskiExI~9B1lbu5cMmc14kq`Y= zzgGRMJ-vI`|5}mUz}6hxoU6C`kLp|RaC)u7L3d}=cXQO;ysI;JDxQ9~-WPGr89k`| zEFYcE?#b$A--92hcW50BMtbcReh)yZEkmuvc+b4)|vM(1Xk=r>=+&)V|}bdgqg^_wMVR z%ifjMjovXjpS`nHcgkI(cZ|-r=5|-T%gNR6`p(|F$)T_kSdjp~%N5?`SqW4El|V}Z z`u8I^h?jEeg_S@hPzh85l|Us>2~+}=KqXKKR05SiB~S@e0+m1|Pzh85l|Us>2~+}= zKqXKKR05SiB~S@e0{2XSe;?#ok*{*M_ucjGY|g#!>z&KqHL5%PzSG}#m;AW1uo9>Q zDuGI%5~u_!fl8nfs01p3N}v*`1S)|_;LHT{e+N7B%DSo&s01p3N}v*`1S)|_pc1GA zDuGI%5~u_!fl8nfs01p3N}v*`1S)|_pc1GADuGI%5~u_!fl8nfs01p3N}v*`1S)|_ zpc1GADuGI%5~u_!fl8nfc+Lde|G!7;#UJ1Q*>iZ#UwOT%5~u_!fl8nfs01p3N}v*` z1S)|_pc1GADuGI%5~u_!fl8nfs01p3N}v*`1S)|_pc1GADuGI%5~u_!fl6Rk0{nY- zyY|=KN}v*`1S)|_pc1GADuGI%5~u_!fin`Qe?C0pJJc1GKqXKKR05SiB~S@e0+m1| jPzjut0ROz$i@b+d*AA|gn<0E!b!L?VA=oIxfCAw>|$vwV3do(gc< zKaf|vz0avS)z#f!Wss${_S$RjbE>+(d#|sPfBTc)|NDpAf4=zhpa1G-zx?&bKmNsM zpMLzaU;XpvKmE~XpZ)09Z(sb`KYsSe@x4D7-@f?Mpa0}$ z`Q6(OKK$nGZ$A3u?LXfB;q5Pe@X_D>*Wdr6bm<`RQN2{q{d^ zS*w@-fi*Wdrw?|k^qSL=NA6@E9k`dMf%^OMu} z;#bH1zx$~2j+$@Y*8Ht~=2m!AZZ+>&^}YABs()|J(f9x8`&H+Ew!KHc&#Lq8)pLIG zz4@d4v(N3UJx9-Tv@g$}{OI}5l7G~_%;r{pI((LUz4p94G50Y0{AT^;?8UR@?%liU zKkGbu=kD#V?&H#LH~T90knc(I>YTH`o~m;8>ZAL1b4GLa`t-wYuT^z_o>O5pC!cfj zs?QwtYLrS%kAp3 z^E&_5d0k&u@Agt3&AIpY=Inl3{axSa{m5B`XU*NZ|FiAuoR7Mm_tE(|FZtQ?bDvk^tL}GIPIbSJ`s?s&-=p+Lxd(f|qt5&6x!3*N zz8RkVInM5XJ@@zd+-KjDz5eb!JNoBxwSP6Y(yN@U`n~2!|+jb3ZzFRnF+V%XfQIuY36VJ@_8&Fz>B;f90s-70Ok|Aw5w(Uvhe) z9D4FjSBIl>`75+XqWsh=UgcNc)0tEC^Qq%7^@^vSd&z_8$1Cjm)N?Mb4$V==E6l#i zH>-OOM*Ftzt8eri=}SJkZ}uf0&DrW3oj2;sJ=|;RcvY`-eP?5LAEWz^`trQ1`?~XV z_1rIc?kg{mFZGJ&9GoAA^6?7!VEQUOebpX(`TdrhZ!aE|Yp*V=9`Eo9wKA1XguSB^} zU0>>1$N3=LUKQq?^kqG{IZz#+Mf2r2Pog?Mq*rL4tY?1m^e5++k5`!e6?d*gx%zNe z?ZK~}sGewk>U+6dXn(vyd#mRjspE;}$;%vi`skUbKXp8FaD5%h;lnGGZ(khBNv!hH zCl_WNS06=vqv$;H^;PFconL>2e4{v;!&l|1!}R6(apz1wF0Vp4S=X;#VK-k+qC7ed z?Su1~r=G|Mv!9+Qw?p&frk|eqxO(>Cd^pqx>HMQ;|Mb(LIqAds?9rh-*!jDBd%!M_ z56Z_O-JYqB&f$}noL(WnJrdRJhgZm#n11`Duj1*?ypCrsKOHxxLifX;C@)c7>Z9CT zs84S0LFdaly+U*3RG9shk8Y3bOP)Slj{WG6E{`8ir3RP+oRIeAzz|A$lqb+;BxRp zb@vU;Q-}I-J~|HRP+xL$p*qwD>HPHM^hEtzG3U(YxSvG1(B6FJsCTH3Pk*9%hw}6% z*RKxwVCvbIoImeFUgf9rC8~E=^;Nn&b^Fg^_T!0Fu07ma<=d;M(QYd(obmV-It2 znEp{N&)mc=$Na?XlbbrO4(ay8Va~(VVfJ@iUt;#z(_F}h%fa>06ZsO;mt3yCQKF#>wfqvrC%c0|m z)&1r?J}8IJ97yM@kPq_V@)Ffy)^T-M>0Mo){cwFyJvkk!%c0|0ued&STn_HJrB5BI zC(48Ar$hZYm(C}r;`HpJXMg(e4l_?~B439&Fa7R?-to+{mp&ZQVfN=-ef0Fp$+|w6 z^~~Xe{5YhWn>jdNh4NtKlgEcAs>{KleG~apmruv}JI)9BVd}~G5}k)`KmCb(9qN-0 z_2GPw4pYy5oDb5SC%Jrjq8vUsFzb9!T`nEUQO6VIK>oz+GY9e~rzdvv(x18dA-_Fw z^$xR7Kg{{5t0(fwhxWmtetoz))DQX9p}HKt4rl%H%)@8V+}s~mmy4SV>3sCW^qHd% zhje}N)FIzqG)InkiR%37$@w53K8yMj-Gli!GzW*N>zA+I@$BP=>X4p$n8%lOIv=cb z^~B80ddG9#EbsE&59G(q!}B~@*AJ^cx_-I1I;?bcm~)fMp(o1WgLG(L9Mbh=KCTYw znS)n8eX}T6A6}vPFn#pQqq`TFK6(7fQ`eu!SD{>e>7ys|r7llBIX^v7Ju&n2xxWfC zCwb=J>8rT8S?_YxJFIfE&;9V@kY1rYe&q09-R+nJ$=db(V-kz`BHc9spGR~A1EKs{W71Q4_3N* z=Fnl*aeG02^XU9Ab$#j`*AMl<)N>y^=VZT}^wIfYrK_8Z^V7{4<#M2Yyh1+n(?^F@ zoi4BAd~)-g`V;k4e)a52KcD%yK03@gt}Y+S!TIqD`Knxfe97rh9nxXyIOKzso^|JU z4s|FeG5uLj&Y$`yulA|*%uQd%v%lh*!JyO#b?i9&(ZU2%{S-XSnXTs z)qebt5ATpaQD5pE-`ZE@R(Yx8F!SiJs;6FYIiv4qm2;GS_Fg&PeK~J(`i0+q1J%7% z`nRXm)vbA3`|3@;xw3Um_Td$p-qkB#>Nw2Y)RX7uN_{JuU-f4_dDVB6zIDDnXust2 z#L73SSH8V-vcKZy!^)TXtY3fjCFg@##}oN09EVmx<1(D^XtMqTZ@2!2Wa&M1*j?w2c>U;F_cKhu;=idHqzp9>dJ6`3`v!6bTT~5|JZhqyX z!|vymJg6_ZoUEr0?@*3hemcGttDMx6SMx@FRX;t?ORuo==U&OH+|p_@I>?I{yA&v>iM40`C+B2Z^f;-`VwdN&|mfKt?cK6+y2saLvsbsy>-cKKC**QXAvoKD}{ zFMl*=)F<~*F!#nQ?Dp2z`P99Kcw*L*^P5|tKBzCT>ZkMLiRv9@AFdCFbXe)?qv$>| zkDfT2oBgsL>7-_W`F@T}7zKcpvi z^}YV=uXyg!@zI>lXHKF%Iu0vcy~FDH`Ta|W`XN7VKl)KdU^PxaTmNpXbefxiIUvI;6wYJKpu_uTa15V?KKFPFIKe z6V(&#LDz?G#VV)LN9TAiM*VwzIWPBDPpszhZLL>*bosfbdZ%X{hjO-}99ZRab$#he z&R1djlly$6j>FOX^y%M=RW5yNe)i)X=DhT))0204*0Ue?dEl>*zpB&u60?qXSmktm z>U;a;bZAcHQy;zWE=OG+q<1*$w||$L^{suGvzq(fTAiDD)%mOWqtAs8nu~YH5BV#e zIzQ~{e5=u1C_njDUEhVbH+l6vRL^PkoUM62zgzpSzBTX7^&HmRJ@@GR zSAQ?=b-vl>#1He=g}(RH?{yAy&YEj}h1XuM`SW1abDhnZ^;hrZqw7bXGxL*o_cQCy zyuJOI=leHxeDyrJ9d>(l{iF4x=49^C_n7lW&wrNh?0wIkf7ZRv&fR)X*_V8FkFEZ* z<{rK8QRmv5TlH7^)_c18Za4bu_wXq1#i~74b4U9}eOsT;(fM2VI=kd#>KkxkvHr`J;PwzE$Vgo3ppSJLl2$+&}rz^H=BFA7*}~ zr|(txqn`Is_d5Ey=ed&~J%2R++Us@$)qUmtbiB&htLNOkd87U2R+u?_sXr@!)bl** zUeEfR&)WOw=XiAfS@%}W&GRO&`cmK8mpMoCReP?=Ir_YrfA-!-&p$dp?{kzt>R#3U zxo`5SZ&p7lcXp4n{MGp@ebt_O^SAbI?OS#3)j6Z*8TGB+@5;N~{8{&KJ@<3(|E%YJ z%^iF0dp{rg;j7&HS?BTZ@g>>^X8kB`USc=Td%(Aqn_Kl)dd}s8>hd9dE6Rs*@WiZF zoDSvSu+yEB57$47*`Itiuk!EB&Hm)o+)iJW)6G#odS2!yw?~EEtE&E}`s_JZpX2QL zv*$T_U++zjv0om7c!jTl+F+mY+3ucF$Mw zzuJ9Q=c)9go?|z!s#p3^dp=9yjld)ht6qu!tDxpwoqdX@WVdUx-Swr{o1S?73`_h8kYa-e%p%z8dI z{?uo=9N6{q!Op*`ZXR5fE2qk>bbS}zZfN!%+$&twpLsa!=FIv!e^>AJIjcUqclx*T z?5oZ{>eD~#*H>YeSJhMZJ#S9dJ6`o|)%8L9CEBag`C#QYukuxW>6Zg%`_i9WZese5 z=FSbb&eIPwhYqVcz4EE=#k22y>;AL%GV9O1X8ULTIj`gLVfrhb&vVBslmpXO>3m(C zFHs+?>Zw;;&g{A6>BnKEtMA3xe114OpC9TQ<>|}*s?JxTKFF6mb@fEPL_Yc`?|fAr z9cB(4s>9j$SRbr%t2+N)ln?jjjrNb;SLUtS(|ma_=kdX;C-?V*Qos1^hDYzAdLETN znp^q0=iAk*+)lsVm$~<%yU)G#qn>lOSI$kYztYt!9PO*#$D=-X)xPgVAAK(GVxU4^z`Ageti|L&a37e zt)F!cbK%x=?(N??x9S_+XVjN-vM+i1@yb`}`Z~=1l#7p|pSz>}NA)=etoC$IolieM4!i!UPyfAeb`O4-dv@ne|0u8K(V;vX(krZd z-Fd1$bM*5;bv#i$Q64?}@bvLlx;m7DSIC#S$+KS{9fy@()hmDM$>mk(9HS_w@|$1P zSLwM&@@@`49Nn8wpE=3tm0vwk4y@|*$~Ri)OSE_DTlrCYZrx|JuiHCwDu3$9GiQ{` zf!SB-=^N$V3z&U5ujBeNui}}fe-&oFJ?OB~)gJ|O&*b(@Jvkq&>Xn{-osWMOGq>Ya zzP`lHKU6^68^F>FfH{{d%P8=gT_Y)%jrhlk**o-Q2F8 z`|&5HFF8LwIX$uRRr+Xu`d0DWEBWjk{jlpF^<~c1zRbaW-YV?oZLN2G_S=f1`Fzfw z^VBBIR}W0&96yM6Zdclqk~nq%Ix;G^G5x1YZk|19_2?b)5T z>%Z!^Z_d^C;JHBg$va*B*|9qJv!CnP_P?IPH-Fw|@1^@WGMo2o?@_n!*7I-e>-Kq; z`rh;J?LYdSy3dXI`TH6Av*YObk3MJa;l8`~C;PfRJAXG%zMMq$d!hOFnm2ks_R0M` zSM}udS-sqgpm-Xb%wH5O`$+zY{d!Kvg?z{5Q6T5riANBD=eI4gpjqb~Q z+#GshSLd&M^hAA$>BG~vm1kb^t+}&(=0g2=;%MD-FO|j_-LK4Li?)Ar$ag) z9fy2fJ^jh$RJuCkgQ??iuWz*9J(vqe`$v6ME`1bpUiNkM^yB&x)uDRwyvOwMSI7@L zUsb19Ik_i)g>ova`Z`@6U-H~j9aeo?>-wPiILx{9d!hNz+~joF)%o(=U0+w13)7E7 zzO6W#lRoE$Tj$UAb@Ou1%-7Ec)7SATr^=&4{oS0b>xWs#NAtTJd86ph-9~fMS8+PD z4-WI(^u+8R<&}>Pa~|H+`9@J5toqdR`J}({(fQ@jp?Y$9g`<9Rs~mdg>*lFLxw!X? z-l0Ce{8gRK*CBtRK6=ObR^jNp^yPf@L_T`O>D_zS`OJa*9i~5d*FU;< z=5(Kn+#|U>I!rz1@WHHSKOf}BJDl}bd0TV3^Hg=ZxykLRj>F2YeiWK}RQ}%FZole2 z_s-Sd&F|`RVCBzq_@2<0m^sPQ-{qx$l;_@~eU&fss&mm{mA6VCox=yK`CYy0%loN( z^lpwi9POLkL+*vQn;X4{v+mj4d@s7a^})_RTkrNUH?f=3-XfV=dH@iJ(F*plYc(5D!1Dw=XU<1>Z{JX>O8CV>F(=Zb8?TI zn>_ui`BC#SH~H#wWM0lop8kuwp&K53AEq5Wr_0Yhl6U>;UEiwjL-QU5t8-WSqwM`? zd(7U~tpBX@KFglf{XE zKY7)6>9?D55AF@p6RYP~^;LT29?fU>sQjtpusTOqulnd!f2HetRDATkbH9$~J@dn? zkMh0!qjM@>e%`n4BL|u%530k|aVQ5?y82OQ-=p%MHP`ci&XJrxi>q>G=T`pNeP;dL zTzR8d<(;kjJUIt$ujG7PT^~OVvp@B$T;`p?$hT~SfV{iZJ^T>OYz4q?0x8HrdI#%~o>9hCY`{r5E z&u8df@Wig3{)@Y@8@}p&IAxyC=OXn{UgeJVU%s!uch`M>{XWimKdR@DIu7r>?^$za z&pEoMpYQxQOdUTOo!1<^!>xXO&>S3Ay80-t%E@_0<>s8T`FlC%=sjQeF?v&%@4A8K zy@C9kC(n~r^-9;T4(YR4^{0-*YL0qh`qevqt509z)_K`?6t{1eU->G%%d6`0;HYoZ zFOPpKuF5fgHZT3j~~l2Cf^p-wpWO-*4i5Cv81n_Tlipr?{qFH*np+bpzK8ytfTyJve(X7`=-ue#SO=lcA8 zc>~vb@Lcb~ZQK8UZa93F=V@PP?~2p&&z1B+z6$wZ`YJtr$+Itc`je+WdFRi1$5-c9 zd6nMn_g?F-a$m1K-fo?0uho{_MTZ`nTTCz2LIJrxJs7?t6cTOS%3B?U!9lvTY0tDtZu$ONQbH8iIqR~ z>`TsnHrgxqNF7hq*Xij?uAdGoeYDQkVK=AiAFbbO&geerd-VNs-%;M3tE-RBc~;-6 zo&V9#Q|(vjXTMjEy4TtF@+$MM=lk4Ga_jTk+V^O;mFGxq-%3}%aO?)IdmOu`%k@|{ zaMpW!^xUKKw?0LC?ZxWf?@9gQwwDn(9=N^-UiEw6_UYdKeD#l_d+hvtkRMM}hwAuhJUV~%zE|Ei z-@v0i$5s2R%K7G7`(w46*VX?x@lxK?4ZOGa!+GK8v!3;j=AP|))pMPFua7?edhXx; zgs%Jf_WOCZ>b;)lsyVB2wmv8G;oh98Uw(yEAHC{3Ten}Lz36zNIvlO@AC28U-TdrN zu0QpP^CwpR)c5vfUd7GHdg>LY-y7|F@408qn?3KN`A6@q+gp7WN9Vs+pZo02U-?qU z{r}_P7v668x`*o?zPy3!J-FV3>pi$W2iNz&^*ylXd*F6^_x~Kt&gH-G+#AU6o7ru$ ze=9dvKR$}1eWN~e`60bSewcoGB46qi=YMwe`@?&m!@b^nu6gD|`g_Gy_pmDGy*`I$ zJ@@t8Z%=Oa9%lW|`hMiz&JDAE@ndda_P$5`qrTBo$%Fcm(-Zk%){o}*ntQJ~d++O6 z_S}2USLwf=*Q-3q+c`#`XXo4dzvCfa4u0X+pVM_8*L{3>16$vN-+%9Re;?jHfBWR) F{{bwmwXFaE literal 0 HcmV?d00001 diff --git a/test/test_restart.py b/test/test_restart.py index cb05ef624..6bd9ba34c 100644 --- a/test/test_restart.py +++ b/test/test_restart.py @@ -74,6 +74,132 @@ def test_restart_cv(actx_factory, nspecies): from mirgecom.restart import read_restart_data restart_data = read_restart_data(actx, rst_filename) - resid = test_state - restart_data["state"] + rst_state = restart_data["state"] + + resid = test_state - rst_state + from mirgecom.simutil import max_component_norm + assert max_component_norm(dcoll, resid, np.inf) == 0 + + +@pytest.mark.parametrize("src_trg_np", [(1, 2), + (1, 4), + (2, 4), + (4, 2), + (4, 1)]) +def test_interdecomp_overlap(src_trg_np): + """Test that restart can read a CV array container.""" + import pickle + from mirgecom.simutil import interdecomposition_overlap + print(f"{src_trg_np=}") + src_np, trg_np = src_trg_np + + trg_decomp_file = f"data/M24k_mesh_decomp_np{trg_np}_pkl_data" + src_decomp_file = f"data/M24k_mesh_decomp_np{src_np}_pkl_data" + + with open(src_decomp_file, "rb") as file: + src_dcmp = pickle.load(file) + with open(trg_decomp_file, "rb") as file: + trg_dcmp = pickle.load(file) + + from mirgecom.simutil import invert_decomp + src_part_els = invert_decomp(src_dcmp) + trg_part_els = invert_decomp(trg_dcmp) + + nsrc_parts = len(src_part_els) + ntrg_parts = len(trg_part_els) + + print(f"Numver of source partitions: {nsrc_parts}.") + print(f"Numver of target partitions: {ntrg_parts}.") + + idx = interdecomposition_overlap(trg_dcmp, src_dcmp) + nsrc_els = len(src_dcmp) + ntrg_els = len(trg_dcmp) + assert nsrc_els == ntrg_els + assert len(idx) == ntrg_parts + + nolap = 0 + for trank in range(ntrg_parts): + for src_rank, olap in idx[trank].items(): + olen = len(olap) + nolap = nolap + olen + print(f"Rank({trank}) olap w/OGRank({src_rank}) is {olen} els.") + assert nolap == nsrc_els + + +def test_dofarray_mapped_copy(actx_factory): + """Test that restart can read a CV array container.""" + actx = actx_factory() + nel_1d = 4 + dim = 3 + from meshmode.mesh.generation import generate_regular_rect_mesh + mesh = generate_regular_rect_mesh( + a=(-0.5,) * dim, b=(0.5,) * dim, nelements_per_axis=(nel_1d,) * dim + ) + order = 3 + dcoll = create_discretization_collection(actx, mesh, order=order) + nodes = actx.thaw(dcoll.nodes()) + + test_data_1 = 1. * nodes[0] + test_data_2 = 2. * nodes[1] + nelems, nnodes = test_data_1[0].shape + + # Copy the whole thing + el_map = {} + for iel in range(nelems): + el_map[iel] = iel + + from mirgecom.simutil import copy_mapped_dof_array_data + test_data_1 = copy_mapped_dof_array_data(test_data_1, test_data_2, el_map) + + # print(f"{test_data_1=}") + # print(f"{test_data_2=}") + # raise AssertionError() + + resid = test_data_1 - test_data_2 + from mirgecom.simutil import max_component_norm + assert max_component_norm(dcoll, resid, np.inf) == 0 + + # Copy half the data + test_data_1 = 3. * nodes[0] + test_data_2 = 4. * nodes[1] + + el_map = {} + ncopy = int(nelems/2) + el_map_1 = {} + el_map_2 = {} + for iel in range(ncopy): + el_map_1[iel] = iel + for iel in range(ncopy, nelems): + el_map_2[iel] = iel + test_data_1 = copy_mapped_dof_array_data(test_data_1, test_data_2, el_map_1) + test_data_2 = copy_mapped_dof_array_data(test_data_2, test_data_1, el_map_2) + + # print(f"{test_data_1=}") + # print(f"{test_data_2=}") + # raise AssertionError() + + resid = test_data_1 - test_data_2 + from mirgecom.simutil import max_component_norm + assert max_component_norm(dcoll, resid, np.inf) == 0 + + test_data_1 = 1. * nodes[0] + 1.0 + test_data_2 = 8. * nodes[1] + 2.0 + + # Copy every other element + el_map_odd = {} + el_map_even = {} + for iel in range(nelems): + if iel % 2: + el_map_odd[iel] = iel + else: + el_map_even[iel] = iel + test_data_1 = copy_mapped_dof_array_data(test_data_1, test_data_2, el_map_odd) + test_data_2 = copy_mapped_dof_array_data(test_data_2, test_data_1, el_map_even) + + # print(f"{test_data_1=}") + # print(f"{test_data_2=}") + # raise AssertionError() + + resid = test_data_1 - test_data_2 from mirgecom.simutil import max_component_norm assert max_component_norm(dcoll, resid, np.inf) == 0 From a7ab1b21cf6c2b03b933fceb611671d6e5ef37db Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 29 Sep 2023 00:15:56 -0500 Subject: [PATCH 15/55] Beef up comments --- mirgecom/restart.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index 3db380152..8d18a2acc 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -75,6 +75,9 @@ def _create_zeros_like_dofarray(actx, nel, ary): zeros_array = actx.zeros((nel, nnodes)) return DOFArray(actx, (zeros_array,)) + # Traverse the restart_data and copy data from the src + # into the target DOFArrays in-place, so that the trg data + # is persistent across multiple calls with different src data. def _recursive_map_and_copy(trg_item, src_item, elem_map): """Recursively map and copy DOFArrays from src_item.""" if isinstance(src_item, DOFArray): @@ -94,8 +97,11 @@ def _recursive_map_and_copy(trg_item, src_item, elem_map): elem_map) for k, v in asdict(src_item).items()}) else: - return src_item + return src_item # dupe non-dof data outright + # Creates a restart data set w/zeros of (trg=part-specific) size for + # all the DOFArray data. To be updated in-place with data from each + # src part. def _recursive_init_with_zeros(sample_item, trg_zeros): """Recursively initialize data structures with zeros or original data.""" black_list = ["volume_to_local_mesh_data", "mesh"] From c0f06e663d8b01b6c408e5e61aea0f328d1d2bb1 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Fri, 29 Sep 2023 08:13:26 -0700 Subject: [PATCH 16/55] Check that the restart path is a directory. --- mirgecom/restart.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index d5b95bdc2..6a28472a0 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -32,6 +32,10 @@ from meshmode.dof_array import array_context_for_pickling +class PathError(RuntimeError): + pass + + def read_restart_data(actx, filename): """Read the raw restart data dictionary from the given pickle restart file.""" with array_context_for_pickling(actx): @@ -47,7 +51,9 @@ def write_restart_file(actx, restart_data, filename, comm=None): if rank == 0: import os rst_dir = os.path.dirname(filename) - if rst_dir: + if os.path.exists(rst_dir) and not os.path.isdir(rst_dir): + raise PathError(f"{rst_dir} exists and is not a directory.") + if not os.path.exists(rst_dir): os.makedirs(rst_dir, exist_ok=True) if comm: comm.barrier() From 9bb89ffdd030d91911ca4f1cfce637ac24bdd3b4 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Sun, 1 Oct 2023 05:46:29 -0700 Subject: [PATCH 17/55] Move redist to restart module. --- bin/redist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/redist.py b/bin/redist.py index 25ca771a8..5cace11c9 100644 --- a/bin/redist.py +++ b/bin/redist.py @@ -304,7 +304,7 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): print(f"Generating the restart data for {ndist} parts...") target_decomp_map_file = mesh_filename + f"_decomp_np{ndist}.pkl" - from mirgecom.simutil import redistribute_restart_data + from mirgecom.restart import redistribute_restart_data redistribute_restart_data(actx, comm, source_decomp_map_file, input_path, target_decomp_map_file, output_path, mesh_filename) From a286c083713ce830b2457ac7b33ad24d3315efc2 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Wed, 4 Oct 2023 06:21:10 -0700 Subject: [PATCH 18/55] Move redist to restart module, correct glob path. --- bin/redist.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/redist.py b/bin/redist.py index 25ca771a8..8788683aa 100644 --- a/bin/redist.py +++ b/bin/redist.py @@ -130,7 +130,10 @@ def main(actx_class, mesh_source=None, ndist=None, mdist=None, if mdist is None: # Try to detect it. If can't then fail. if rank == 0: - files = glob.glob(input_path) + search_pattern = input_path + "*" + print(f"Searching input path {search_pattern}.") + files = glob.glob(search_pattern) + print(f"Found files: {files}") xps = ["_decomp_", "_mesh_"] ffiles = [f for f in files if not any(xc in f for xc in xps)] mdist = len(ffiles) @@ -304,7 +307,7 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): print(f"Generating the restart data for {ndist} parts...") target_decomp_map_file = mesh_filename + f"_decomp_np{ndist}.pkl" - from mirgecom.simutil import redistribute_restart_data + from mirgecom.restart import redistribute_restart_data redistribute_restart_data(actx, comm, source_decomp_map_file, input_path, target_decomp_map_file, output_path, mesh_filename) From 08622ddbd94949eaa80855bc0ff9e1ab4c176dc9 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Wed, 4 Oct 2023 06:35:30 -0700 Subject: [PATCH 19/55] Use array context for picklin, zeros_like, debugging diagnostics --- mirgecom/restart.py | 55 +++++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index 6b3f6b41f..49f5ff40c 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -78,7 +78,7 @@ def redistribute_restart_data(actx, comm, source_decomp_map_file, input_path, def _create_zeros_like_dofarray(actx, nel, ary): # Get nnodes from the shape of the sample DOFArray _, nnodes = ary[0].shape - zeros_array = actx.zeros((nel, nnodes)) + zeros_array = actx.zeros(shape=(nel, nnodes), dtype=ary[0].dtype) return DOFArray(actx, (zeros_array,)) # Traverse the restart_data and copy data from the src @@ -86,6 +86,14 @@ def _create_zeros_like_dofarray(actx, nel, ary): # is persistent across multiple calls with different src data. def _recursive_map_and_copy(trg_item, src_item, elem_map): """Recursively map and copy DOFArrays from src_item.""" + if trg_item is None: + print(f"{src_item=}") + raise ValueError("trg_item is None, but src_item is not.") + if src_item is None: + print(f"{trg_item=}") + raise ValueError("src_item is None, but trg_item is not.") + # print(f"{trg_item=}") + # print(f"{src_item=}") if isinstance(src_item, DOFArray): if trg_item is None: raise ValueError("No corresponding target DOFArray found.") @@ -158,14 +166,16 @@ def _recursive_init_with_zeros(sample_item, trg_zeros): # Read one source restart file to get the 'order' and a sample DOFArray mesh_data_item = "volume_to_local_mesh_data" - sample_restart_file = f"{input_path}={writer_rank:04d}.pkl" - with open(sample_restart_file, "rb") as f: - sample_data = pickle.load(f) - if "mesh" in sample_data: - mesh_data_item = "mesh" # check mesh data return type instead - sample_dof_array = \ - next(val for key, val in - sample_data.items() if isinstance(val, DOFArray)) + sample_restart_file = f"{input_path}-{writer_rank:04d}.pkl" + with array_context_for_pickling(actx): + with open(sample_restart_file, "rb") as f: + sample_data = pickle.load(f) + if "mesh" in sample_data: + mesh_data_item = "mesh" # check mesh data return type instead + + sample_dof_array = \ + next(val for key, val in + sample_data.items() if isinstance(val, DOFArray)) for trg_part, olaps in xdo.items(): # Number of elements in each target partition @@ -177,26 +187,33 @@ def _recursive_init_with_zeros(sample_item, trg_zeros): actx, trg_nel, sample_dof_array) # Use trg_zeros to reset all dof arrays in the out_rst_data to # the current (trg_part) size made of zeros. - out_rst_data = _recursive_init_with_zeros(sample_data, trg_zeros) + with array_context_for_pickling(actx): + out_rst_data = _recursive_init_with_zeros(sample_data, trg_zeros) # Read and Map DOFArrays from source to target for src_part, elem_map in olaps.items(): # elem_map={trg-local-index : src-local-index} for trg,src parts src_restart_file = f"{input_path}-{src_part:04d}.pkl" - with open(src_restart_file, "rb") as f: - src_rst_data = pickle.load(f) - out_rst_data = _recursive_map_and_copy( - out_rst_data, src_rst_data, elem_map) + with array_context_for_pickling(actx): + with open(src_restart_file, "rb") as f: + src_rst_data = pickle.load(f) + src_rst_data.pop("volume_to_local_mesh_data", None) + src_rst_data.pop("mesh", None) + with array_context_for_pickling(actx): + out_rst_data = _recursive_map_and_copy( + out_rst_data, src_rst_data, elem_map) # Read new mesh data and stack it in the restart file mesh_pkl_filename = f"{mesh_filename}_np{trg_nparts}_rank{trg_part}.pkl" - with open(mesh_pkl_filename, "rb") as pkl_file: - global_nelements, volume_to_local_mesh_data = pickle.load(pkl_file) - out_rst_data[mesh_data_item] = volume_to_local_mesh_data + with array_context_for_pickling(actx): + with open(mesh_pkl_filename, "rb") as pkl_file: + global_nelements, volume_to_local_mesh_data = pickle.load(pkl_file) + out_rst_data[mesh_data_item] = volume_to_local_mesh_data # Write out the trg_part-specific redistributed pkl restart file output_filename = f"{output_path}-{trg_part:04d}.pkl" - with open(output_filename, "wb") as f: - pickle.dump(out_rst_data, f) + with array_context_for_pickling(actx): + with open(output_filename, "wb") as f: + pickle.dump(out_rst_data, f) return From cdded96f1ca916aa1f6db6c2aadbfe125b4dcce7 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Wed, 4 Oct 2023 06:36:35 -0700 Subject: [PATCH 20/55] Get mesh on reader_rank 0, and share some timings with the user. --- mirgecom/simutil.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index daf309641..0cfcbaef9 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -1207,6 +1207,7 @@ def distribute_mesh_pkl(comm, get_mesh_data, filename="mesh", The number of elements in the global mesh """ from mpi4py.util import pkl5 + from datetime import datetime comm_wrapper = pkl5.Intracomm(comm) num_ranks = comm_wrapper.Get_size() @@ -1246,12 +1247,21 @@ def partition_generator_func(mesh, tag_to_elements, num_target_ranks): from meshmode.distributed import get_partition_by_pymetis return get_partition_by_pymetis(mesh, num_target_ranks) - if logmgr: - logmgr.add_quantity(t_mesh_data) - with t_mesh_data.get_sub_timer(): + if reader_rank == 0: + if logmgr: + logmgr.add_quantity(t_mesh_data) + with t_mesh_data.get_sub_timer(): + global_data = get_mesh_data() + else: global_data = get_mesh_data() + global_data = reader_comm_wrapper.bcast(global_data, root=0) else: - global_data = get_mesh_data() + global_data = reader_comm_wrapper.bcast(None, root=0) + + reader_comm.Barrier() + if reader_rank == 0: + print(f"{datetime.now()}: Done reading source mesh from file." + " Partitioning...") from meshmode.mesh import Mesh if isinstance(global_data, Mesh): @@ -1272,6 +1282,10 @@ def partition_generator_func(mesh, tag_to_elements, num_target_ranks): rank_per_element = partition_generator_func(mesh, tag_to_elements, num_target_ranks) + reader_comm.Barrier() + if reader_rank == 0: + print(f"{datetime.now()}: Done partitioning mesh. Splitting...") + # Save this little puppy for later (m-to-n restart support) if reader_rank == 0: part_table_fname = filename + f"_decomp_np{num_target_ranks}.pkl" @@ -1298,6 +1312,10 @@ def get_rank_to_mesh_data(): else: rank_to_mesh_data = get_rank_to_mesh_data() + reader_comm.Barrier() + if reader_rank == 0: + print(f"{datetime.now()}: Done splitting mesh. Writing...") + if logmgr: logmgr.add_quantity(t_mesh_dist) with t_mesh_dist.get_sub_timer(): @@ -1318,6 +1336,10 @@ def get_rank_to_mesh_data(): with open(pkl_filename, "wb") as pkl_file: pickle.dump(mesh_data_to_pickle, pkl_file) + reader_comm.Barrier() + if reader_rank == 0: + print(f"{datetime.now()}: Done writing partitioned mesh.") + def extract_volumes(mesh, tag_to_elements, selected_tags, boundary_tag): r""" From 9a2d8a75257f686df0d02f46de23e30ba0ef74b0 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 5 Oct 2023 09:14:16 -0500 Subject: [PATCH 21/55] add options to specify target decomp map --- bin/redist.py | 48 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/bin/redist.py b/bin/redist.py index 8788683aa..9d1bb7c80 100644 --- a/bin/redist.py +++ b/bin/redist.py @@ -78,6 +78,8 @@ class MyRuntimeError(RuntimeError): def main(actx_class, mesh_source=None, ndist=None, mdist=None, output_path=None, input_path=None, log_path=None, casename=None, use_1d_part=None, use_wall=False, + source_decomp_map_file=None, + target_decomp_map_file=None, restart_file=None): """Redistribute a mirgecom restart dataset.""" if mesh_source is None: @@ -174,6 +176,20 @@ def main(actx_class, mesh_source=None, ndist=None, mdist=None, raise ApplicationOptionsError("Output path must be different than input" " location because of filename collisions.") + # Try to detect whether target mesh is already created, will be when + # multiple data transfers are being done. + if target_decomp_map_file is None: + target_decomp_map_file = \ + output_directory + "/" + casename + f"_decomp_np{ndist}.pkl" + if os.path.exists(target_decomp_map_file): + trg_dir, trg_map_file = os.path.split(target_decomp_map_file) + # Use regex to extract casename + match = re.match(r"(.+)_decomp\.pkl", trg_map_file) + if not match: + match = re.math(r"(.+).pkl", trg_map_file) + mesh_casename = match + + decomp_map_file_search_pattern = \ input_data_directory + f"/*_decomp_np{mdist}.pkl" input_decomp_map_files = glob.glob(decomp_map_file_search_pattern) @@ -292,19 +308,21 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): comm.Barrier() - if rank == 0: - print(f"Partitioning mesh to {ndist} parts, writing to {mesh_filename}...") + if generate_target_mesh: + if rank == 0: + print(f"Partitioning mesh to {ndist} parts, writing to {mesh_filename}...") - # This bit creates the N-parted mesh pkl files and partition table - distribute_mesh_pkl( - comm, get_mesh_data, filename=mesh_filename, - num_target_ranks=ndist, partition_generator_func=part_func, logmgr=logmgr) + # This bit creates the N-parted mesh pkl files and partition table + distribute_mesh_pkl( + comm, get_mesh_data, filename=mesh_filename, + num_target_ranks=ndist, partition_generator_func=part_func, logmgr=logmgr) + + comm.Barrier() - comm.Barrier() + if rank == 0: + print("Done partitioning target mesh, mesh pkl files written.") - if rank == 0: - print("Done partitioning target mesh, mesh pkl files written.") - print(f"Generating the restart data for {ndist} parts...") + print(f"Generating the restart data for {ndist} parts...") target_decomp_map_file = mesh_filename + f"_decomp_np{ndist}.pkl" from mirgecom.restart import redistribute_restart_data @@ -341,6 +359,12 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): parser.add_argument("-m", "--mdist", type=int, dest="mdist", nargs="?", action="store", help="Number of source data parts") + parser.add_argument("-t", "--partition-table-out", dest="npart", + nargs="?", action="store", + help="Target partition table pkl file.") + parser.add_argument("-p", "--partition-table-in", dest="mpart", + nargs="?", action="store", + help="Source partition table pkl file.") parser.add_argument("-s", "--source", type=str, dest="source", nargs="?", action="store", help="Gmsh mesh source file") parser.add_argument("-o", "--ouput-path", type=str, dest="output_path", @@ -365,4 +389,6 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): output_path=args.output_path, ndist=args.ndist, input_path=args.input_path, mdist=args.mdist, log_path=args.log_path, casename=args.casename, - use_1d_part=args.one_d_part, use_wall=args.use_wall) + use_1d_part=args.one_d_part, + source_decomp_map_file=args.mpart, + target_decomp_map_file=args.npart, use_wall=args.use_wall) From 5bf5d585915ffe304baec4ecb5bcb36d6ae32c63 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Thu, 5 Oct 2023 21:54:30 -0500 Subject: [PATCH 22/55] Refactor for access to multivol partition datastructure --- mirgecom/simutil.py | 82 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 18 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 0cfcbaef9..84b6ce927 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -904,19 +904,16 @@ def _partition_single_volume_mesh( mesh, rank_to_elements, return_parts=return_ranks) -def _partition_multi_volume_mesh( - mesh, num_ranks, rank_per_element, tag_to_elements, volume_to_tags, *, - return_ranks=None): - if return_ranks is None: - return_ranks = list(range(num_ranks)) +def _get_multi_volume_partitions(mesh, num_ranks, elements_to_rank, + tag_to_elements, volume_to_tags): + + volumes = list(volume_to_tags.keys()) tag_to_volume = { tag: vol for vol, tags in volume_to_tags.items() for tag in tags} - volumes = list(volume_to_tags.keys()) - volume_index_per_element = np.full(mesh.nelements, -1, dtype=int) for tag, elements in tag_to_elements.items(): volume_index_per_element[elements] = volumes.index( @@ -929,12 +926,28 @@ def _partition_multi_volume_mesh( PartID(volumes[vol_idx], rank): np.where( (volume_index_per_element == vol_idx) - & (rank_per_element == rank))[0] + & (elements_to_rank == rank))[0] for vol_idx in range(len(volumes)) for rank in range(num_ranks)} + return part_id_to_elements + + +def _partition_multi_volume_mesh( + mesh, num_ranks, rank_per_element, + part_id_to_elements, tag_to_elements, volume_to_tags, *, + return_ranks=None): + if return_ranks is None: + return_ranks = list(range(num_ranks)) + volumes = list(volume_to_tags.keys()) + # TODO: Add a public meshmode function to accomplish this? So we're # not depending on meshmode internals + # PartID is (rank, vol) pair + # part_index is range(len(PartID)) + # part_id_to_part_index is {PartID: part_index} + # global_elem_to_part_elem is ary[global_elem_index] = \ + # [part_index, local_element_index] part_id_to_part_index = { part_id: part_index for part_index, part_id in enumerate(part_id_to_elements.keys())} @@ -943,6 +956,8 @@ def _partition_multi_volume_mesh( mesh.nelements, part_id_to_elements, part_id_to_part_index, mesh.element_id_dtype) + # tag_to_global_to_part = \ + # {tag: ary[global_elem_index][part_index, local_element_index]} tag_to_global_to_part = { tag: global_elem_to_part_elem[elements, :] for tag, elements in tag_to_elements.items()} @@ -1112,9 +1127,12 @@ def get_rank_to_mesh_data(): mesh, num_ranks, rank_per_element, return_ranks=node_ranks) else: - rank_to_mesh_data = _partition_multi_volume_mesh( + part_id_to_elements = _get_multi_volume_partitions( mesh, num_ranks, rank_per_element, tag_to_elements, - volume_to_tags, return_ranks=node_ranks) + volume_to_tags) + rank_to_mesh_data = _partition_multi_volume_mesh( + mesh, num_ranks, rank_per_element, part_id_to_elements, + tag_to_elements, volume_to_tags, return_ranks=node_ranks) rank_to_node_rank = { rank: node_rank @@ -1239,8 +1257,13 @@ def distribute_mesh_pkl(comm, get_mesh_data, filename="mesh", my_ending_rank = my_starting_rank + num_ranks_this_reader - 1 ranks_to_write = list(range(my_starting_rank, my_ending_rank+1)) + if reader_rank == 0: + print("Reading(world_rank,reader_rank): " + "Writing[starting_rank,ending_rank]") + print("----------------------------------") + reader_comm.Barrier() print(f"R({my_rank},{reader_rank}): " - f"W({my_starting_rank},{my_ending_rank})") + f"W[{my_starting_rank},{my_ending_rank}]") if partition_generator_func is None: def partition_generator_func(mesh, tag_to_elements, num_target_ranks): @@ -1254,13 +1277,15 @@ def partition_generator_func(mesh, tag_to_elements, num_target_ranks): global_data = get_mesh_data() else: global_data = get_mesh_data() + print(f"{datetime.now()}: Done reading source mesh from file. " + "Broadcasting...") global_data = reader_comm_wrapper.bcast(global_data, root=0) else: global_data = reader_comm_wrapper.bcast(None, root=0) reader_comm.Barrier() if reader_rank == 0: - print(f"{datetime.now()}: Done reading source mesh from file." + print(f"{datetime.now()}: Done distrbuting source mesh data." " Partitioning...") from meshmode.mesh import Mesh @@ -1282,10 +1307,6 @@ def partition_generator_func(mesh, tag_to_elements, num_target_ranks): rank_per_element = partition_generator_func(mesh, tag_to_elements, num_target_ranks) - reader_comm.Barrier() - if reader_rank == 0: - print(f"{datetime.now()}: Done partitioning mesh. Splitting...") - # Save this little puppy for later (m-to-n restart support) if reader_rank == 0: part_table_fname = filename + f"_decomp_np{num_target_ranks}.pkl" @@ -1294,6 +1315,30 @@ def partition_generator_func(mesh, tag_to_elements, num_target_ranks): with open(part_table_fname, "wb") as pkl_file: pickle.dump(rank_per_element, pkl_file) + reader_comm.Barrier() + if reader_rank == 0: + print(f"{datetime.now()}: Done with global partitioning. Splitting...") + + if tag_to_elements is None: + part_id_to_elements = None + else: + part_id_to_elements = _get_multi_volume_partitions( + mesh, num_ranks, rank_per_element, tag_to_elements, + volume_to_tags) + # Save this little puppy for later (m-to-n restart support) + if reader_rank == 0: + mv_part_table_fname = \ + filename + f"_multivol_decomp_np{num_target_ranks}.pkl" + if os.path.exists(mv_part_table_fname): + os.remove(mv_part_table_fname) + with open(mv_part_table_fname, "wb") as pkl_file: + pickle.dump(part_id_to_elements, pkl_file) + + reader_comm.Barrier() + if reader_rank == 0: + print(f"{datetime.now()}: - Got PartID-to-elements. " + "Making mesh data structures...") + def get_rank_to_mesh_data(): if tag_to_elements is None: rank_to_mesh_data = _partition_single_volume_mesh( @@ -1301,10 +1346,11 @@ def get_rank_to_mesh_data(): return_ranks=ranks_to_write) else: rank_to_mesh_data = _partition_multi_volume_mesh( - mesh, num_target_ranks, rank_per_element, tag_to_elements, - volume_to_tags, return_ranks=ranks_to_write) + mesh, num_ranks, rank_per_element, part_id_to_elements, + tag_to_elements, volume_to_tags, return_ranks=ranks_to_write) return rank_to_mesh_data + reader_comm.Barrier() if logmgr: logmgr.add_quantity(t_mesh_split) with t_mesh_split.get_sub_timer(): From a5fba6b64905247cc139a6957590805777f8d536 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 6 Oct 2023 10:49:50 -0500 Subject: [PATCH 23/55] Fix bug in refactored --- mirgecom/simutil.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 84b6ce927..4fc7b8449 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -904,7 +904,7 @@ def _partition_single_volume_mesh( mesh, rank_to_elements, return_parts=return_ranks) -def _get_multi_volume_partitions(mesh, num_ranks, elements_to_rank, +def _get_multi_volume_partitions(mesh, num_ranks, rank_per_element, tag_to_elements, volume_to_tags): volumes = list(volume_to_tags.keys()) @@ -926,7 +926,7 @@ def _get_multi_volume_partitions(mesh, num_ranks, elements_to_rank, PartID(volumes[vol_idx], rank): np.where( (volume_index_per_element == vol_idx) - & (elements_to_rank == rank))[0] + & (rank_per_element == rank))[0] for vol_idx in range(len(volumes)) for rank in range(num_ranks)} @@ -1323,7 +1323,7 @@ def partition_generator_func(mesh, tag_to_elements, num_target_ranks): part_id_to_elements = None else: part_id_to_elements = _get_multi_volume_partitions( - mesh, num_ranks, rank_per_element, tag_to_elements, + mesh, num_target_ranks, rank_per_element, tag_to_elements, volume_to_tags) # Save this little puppy for later (m-to-n restart support) if reader_rank == 0: @@ -1346,7 +1346,7 @@ def get_rank_to_mesh_data(): return_ranks=ranks_to_write) else: rank_to_mesh_data = _partition_multi_volume_mesh( - mesh, num_ranks, rank_per_element, part_id_to_elements, + mesh, num_target_ranks, rank_per_element, part_id_to_elements, tag_to_elements, volume_to_tags, return_ranks=ranks_to_write) return rank_to_mesh_data From b4328bb962a961f7d74eb45aaf118f784bc60a26 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Fri, 6 Oct 2023 10:52:50 -0500 Subject: [PATCH 24/55] Back out changes to redist, for now. --- bin/redist.py | 48 +++++++++++------------------------------------- 1 file changed, 11 insertions(+), 37 deletions(-) diff --git a/bin/redist.py b/bin/redist.py index 9d1bb7c80..8788683aa 100644 --- a/bin/redist.py +++ b/bin/redist.py @@ -78,8 +78,6 @@ class MyRuntimeError(RuntimeError): def main(actx_class, mesh_source=None, ndist=None, mdist=None, output_path=None, input_path=None, log_path=None, casename=None, use_1d_part=None, use_wall=False, - source_decomp_map_file=None, - target_decomp_map_file=None, restart_file=None): """Redistribute a mirgecom restart dataset.""" if mesh_source is None: @@ -176,20 +174,6 @@ def main(actx_class, mesh_source=None, ndist=None, mdist=None, raise ApplicationOptionsError("Output path must be different than input" " location because of filename collisions.") - # Try to detect whether target mesh is already created, will be when - # multiple data transfers are being done. - if target_decomp_map_file is None: - target_decomp_map_file = \ - output_directory + "/" + casename + f"_decomp_np{ndist}.pkl" - if os.path.exists(target_decomp_map_file): - trg_dir, trg_map_file = os.path.split(target_decomp_map_file) - # Use regex to extract casename - match = re.match(r"(.+)_decomp\.pkl", trg_map_file) - if not match: - match = re.math(r"(.+).pkl", trg_map_file) - mesh_casename = match - - decomp_map_file_search_pattern = \ input_data_directory + f"/*_decomp_np{mdist}.pkl" input_decomp_map_files = glob.glob(decomp_map_file_search_pattern) @@ -308,21 +292,19 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): comm.Barrier() - if generate_target_mesh: - if rank == 0: - print(f"Partitioning mesh to {ndist} parts, writing to {mesh_filename}...") + if rank == 0: + print(f"Partitioning mesh to {ndist} parts, writing to {mesh_filename}...") - # This bit creates the N-parted mesh pkl files and partition table - distribute_mesh_pkl( - comm, get_mesh_data, filename=mesh_filename, - num_target_ranks=ndist, partition_generator_func=part_func, logmgr=logmgr) - - comm.Barrier() + # This bit creates the N-parted mesh pkl files and partition table + distribute_mesh_pkl( + comm, get_mesh_data, filename=mesh_filename, + num_target_ranks=ndist, partition_generator_func=part_func, logmgr=logmgr) - if rank == 0: - print("Done partitioning target mesh, mesh pkl files written.") + comm.Barrier() - print(f"Generating the restart data for {ndist} parts...") + if rank == 0: + print("Done partitioning target mesh, mesh pkl files written.") + print(f"Generating the restart data for {ndist} parts...") target_decomp_map_file = mesh_filename + f"_decomp_np{ndist}.pkl" from mirgecom.restart import redistribute_restart_data @@ -359,12 +341,6 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): parser.add_argument("-m", "--mdist", type=int, dest="mdist", nargs="?", action="store", help="Number of source data parts") - parser.add_argument("-t", "--partition-table-out", dest="npart", - nargs="?", action="store", - help="Target partition table pkl file.") - parser.add_argument("-p", "--partition-table-in", dest="mpart", - nargs="?", action="store", - help="Source partition table pkl file.") parser.add_argument("-s", "--source", type=str, dest="source", nargs="?", action="store", help="Gmsh mesh source file") parser.add_argument("-o", "--ouput-path", type=str, dest="output_path", @@ -389,6 +365,4 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): output_path=args.output_path, ndist=args.ndist, input_path=args.input_path, mdist=args.mdist, log_path=args.log_path, casename=args.casename, - use_1d_part=args.one_d_part, - source_decomp_map_file=args.mpart, - target_decomp_map_file=args.npart, use_wall=args.use_wall) + use_1d_part=args.one_d_part, use_wall=args.use_wall) From 2445a037adccdf850b86b67fb50ef7c2dce7c7a2 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Mon, 9 Oct 2023 14:24:44 -0500 Subject: [PATCH 25/55] Add overlap mapping utility for multi-volume datasets. --- mirgecom/simutil.py | 73 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 72 insertions(+), 1 deletion(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 4fc7b8449..eccabfa93 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -26,6 +26,11 @@ .. autofunction:: distribute_mesh .. autofunction:: get_number_of_tetrahedron_nodes .. autofunction:: get_box_mesh +.. autofunction:: interdecomposition_mapping +.. autofunction:: interdecomposition_overlap +.. autofunction:: invert_decomp +.. autofunction:: multivolume_interdecomposition_overlap +.. autofunction:: copy_mapped_dof_array_data Simulation support utilities ---------------------------- @@ -1475,7 +1480,6 @@ def copy_mapped_dof_array_data(trg_dof_array, src_dof_array, index_map): for trg_el, src_el in index_map.items(): trg_array[trg_el] = src_array[src_el] - # trg_dof_array_np[0] = trg_array return actx.from_numpy(trg_dof_array_np) @@ -1558,6 +1562,73 @@ def interdecomposition_overlap(target_decomp_map, source_decomp_map, return overlap_maps +# Interdecomposition overlap utility for multi-volume datasets +def multivolume_interdecomposition_overlap(target_decomp, source_decomp, + target_vol_decomp, source_vol_decomp, + return_ranks=None): + """ + Map element indices for overlapping, disparate decompositions with volumes. + + Parameters + ---------- + target_decomp: Decomposition map of the target {rank: [elements]} + source_decomp: Decomposition map of the source {rank: [elements]} + target_vol_decomp: Decomposition map of the target with volumes + {PartID: [elements]} + source_vol_decomp: Decomposition map of the source with volumes + {PartID: [elements]} + return_ranks: List of ranks for which the overlaps should be computed. + If None, all ranks are considered. + + Returns + ------- + A dictionary with structure: + { + target_rank: { + source_rank: { + PartID: { + target_local_index: source_local_index + } + } + } + } + """ + # If no specific ranks are provided, consider all ranks in the target decomp + if return_ranks is None: + return_ranks = list(target_decomp.keys()) + + rank_overlap_map = interdecomposition_mapping(target_decomp, source_decomp) + + overlap_maps = {} + + for trg_rank in return_ranks: + # Extract set of unique PartIDs for current trg rank from tgt_vol_decomp + targ_partids = [partid for partid in target_vol_decomp.keys() + if partid.rank == trg_rank] + + overlap_maps[trg_rank] = {} + for src_rank in rank_overlap_map[trg_rank]: + overlap_maps[trg_rank][src_rank] = {} + + for targ_partid in targ_partids: + src_partid = PartID(volume_tag=targ_partid.volume_tag, rank=src_rank) + overlap_maps[trg_rank][src_rank][targ_partid] = {} + + # Determine element overlaps, set is used for performance considerations + target_elements_set = set(target_vol_decomp[targ_partid]) + source_elements_set = set(source_vol_decomp.get(src_partid, [])) + + common_elements_set = \ + target_elements_set.intersection(source_elements_set) + + for trg_el in common_elements_set: + trg_local_idx = target_vol_decomp[targ_partid].index(trg_el) + src_local_idx = source_vol_decomp[src_partid].index(trg_el) + overlap_maps[trg_rank][src_rank][targ_partid][trg_local_idx] = src_local_idx + + return overlap_maps + + def boundary_report(dcoll, boundaries, outfile_name, *, dd=DD_VOLUME_ALL, mesh=None): """Generate a report of the mesh boundaries.""" From 51f53050f974584a7dbc40cff015be36cfe1f6b6 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Mon, 9 Oct 2023 16:23:30 -0500 Subject: [PATCH 26/55] Util for overlap compute of inverted decomps, convenience --- mirgecom/simutil.py | 79 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 17 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index eccabfa93..ebf469989 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -898,6 +898,17 @@ def generate_and_distribute_mesh(comm, generate_mesh, **kwargs): return distribute_mesh(comm, generate_mesh) +def invert_decomp(decomp_map): + """Return a list of global elements for each partition.""" + from collections import defaultdict + global_elements_per_part = defaultdict(list) + + for elemid, part in enumerate(decomp_map): + global_elements_per_part[part].append(elemid) + + return global_elements_per_part + + def _partition_single_volume_mesh( mesh, num_ranks, rank_per_element, *, return_ranks=None): rank_to_elements = { @@ -1319,6 +1330,10 @@ def partition_generator_func(mesh, tag_to_elements, num_target_ranks): os.remove(part_table_fname) with open(part_table_fname, "wb") as pkl_file: pickle.dump(rank_per_element, pkl_file) + rank_to_elems = invert_decomp(rank_per_element) + part_table_fname = filename + f"_idecomp_np{num_target_ranks}.pkl" + with open(part_table_fname, "wb") as pkl_file: + pickle.dump(rank_to_elems, pkl_file) reader_comm.Barrier() if reader_rank == 0: @@ -1333,7 +1348,7 @@ def partition_generator_func(mesh, tag_to_elements, num_target_ranks): # Save this little puppy for later (m-to-n restart support) if reader_rank == 0: mv_part_table_fname = \ - filename + f"_multivol_decomp_np{num_target_ranks}.pkl" + filename + f"_multivol_idecomp_np{num_target_ranks}.pkl" if os.path.exists(mv_part_table_fname): os.remove(mv_part_table_fname) with open(mv_part_table_fname, "wb") as pkl_file: @@ -1483,6 +1498,47 @@ def copy_mapped_dof_array_data(trg_dof_array, src_dof_array, index_map): return actx.from_numpy(trg_dof_array_np) +def interdecomposition_imapping(target_idecomp, source_idecomp): + """ + Return a mapping of which partitions to source for the target decomp. + + Expects input format: {rank: [elements]} + + Parameters + ---------- + target_idecomp: dict + Target decomposition in the format {rank: [elements]} + source_idecomp: dict + Source decomposition in the format {rank: [elements]} + + Returns + ------- + dict + Dictionary like {trg_rank: [src_ranks]} + """ + from collections import defaultdict + + # Convert {rank: [elements]} format into {element: rank} for faster look-up + source_elem_to_rank = {} + for rank, elements in source_idecomp.items(): + for elem in elements: + source_elem_to_rank[elem] = rank + + interdecomp_map = defaultdict(set) + + for trg_rank, trg_elements in target_idecomp.items(): + for elem in trg_elements: + src_rank = source_elem_to_rank.get(elem) + if src_rank is not None: + interdecomp_map[trg_rank].add(src_rank) + + # Convert sets to lists for the final output + for rank in interdecomp_map: + interdecomp_map[rank] = list(interdecomp_map[rank]) + + return interdecomp_map + + def interdecomposition_mapping(target_decomp, source_decomp): """Return a mapping of which partitions to source for the target decomp.""" from collections import defaultdict @@ -1498,17 +1554,6 @@ def interdecomposition_mapping(target_decomp, source_decomp): return interdecomp_map -def invert_decomp(decomp_map): - """Return a list of global elements for each partition.""" - from collections import defaultdict - global_elements_per_part = defaultdict(list) - - for elemid, part in enumerate(decomp_map): - global_elements_per_part[part].append(elemid) - - return global_elements_per_part - - # Need a function to determine which of my local elements overlap # with a disparate decomp part. Optionally restrict attention to # selected parts. @@ -1563,7 +1608,7 @@ def interdecomposition_overlap(target_decomp_map, source_decomp_map, # Interdecomposition overlap utility for multi-volume datasets -def multivolume_interdecomposition_overlap(target_decomp, source_decomp, +def multivolume_interdecomposition_overlap(target_idecomp, source_idecomp, target_vol_decomp, source_vol_decomp, return_ranks=None): """ @@ -1571,8 +1616,8 @@ def multivolume_interdecomposition_overlap(target_decomp, source_decomp, Parameters ---------- - target_decomp: Decomposition map of the target {rank: [elements]} - source_decomp: Decomposition map of the source {rank: [elements]} + target_idecomp: Decomposition map of the target {rank: [elements]} + source_idecomp: Decomposition map of the source {rank: [elements]} target_vol_decomp: Decomposition map of the target with volumes {PartID: [elements]} source_vol_decomp: Decomposition map of the source with volumes @@ -1595,9 +1640,9 @@ def multivolume_interdecomposition_overlap(target_decomp, source_decomp, """ # If no specific ranks are provided, consider all ranks in the target decomp if return_ranks is None: - return_ranks = list(target_decomp.keys()) + return_ranks = list(target_idecomp.keys()) - rank_overlap_map = interdecomposition_mapping(target_decomp, source_decomp) + rank_overlap_map = interdecomposition_imapping(target_idecomp, source_idecomp) overlap_maps = {} From d995e65bddb58f27b42c3972ad5705fa6538b755 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Mon, 9 Oct 2023 16:24:11 -0500 Subject: [PATCH 27/55] Add some tests of multivol overlap mapping. --- test/test_restart.py | 158 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 155 insertions(+), 3 deletions(-) diff --git a/test/test_restart.py b/test/test_restart.py index 6bd9ba34c..f97bf450e 100644 --- a/test/test_restart.py +++ b/test/test_restart.py @@ -33,7 +33,7 @@ from meshmode.array_context import ( # noqa pytest_generate_tests_for_pyopencl_array_context as pytest_generate_tests) - +from grudge.discretization import PartID logger = logging.getLogger(__name__) @@ -105,11 +105,18 @@ def test_interdecomp_overlap(src_trg_np): src_part_els = invert_decomp(src_dcmp) trg_part_els = invert_decomp(trg_dcmp) + trg_decomp_file = f"data/M24k_mesh_idecomp_np{trg_np}_pkl_data" + src_decomp_file = f"data/M24k_mesh_idecomp_np{src_np}_pkl_data" + with open(src_decomp_file, "wb") as file: + pickle.dump(src_part_els, file) + with open(trg_decomp_file, "wb") as file: + pickle.dump(trg_part_els, file) + nsrc_parts = len(src_part_els) ntrg_parts = len(trg_part_els) - print(f"Numver of source partitions: {nsrc_parts}.") - print(f"Numver of target partitions: {ntrg_parts}.") + print(f"Number of source partitions: {nsrc_parts}.") + print(f"Number of target partitions: {ntrg_parts}.") idx = interdecomposition_overlap(trg_dcmp, src_dcmp) nsrc_els = len(src_dcmp) @@ -203,3 +210,148 @@ def test_dofarray_mapped_copy(actx_factory): resid = test_data_1 - test_data_2 from mirgecom.simutil import max_component_norm assert max_component_norm(dcoll, resid, np.inf) == 0 + + +def test_multivolume_interdecomp_overlap_basic(): + """Test the multivolume_interdecomp_overlap.""" + # Total elements in the testing mesh + total_elements = 100 + + # Create test decomps + src_np, trg_np = 3, 2 + elements_per_rank_src = total_elements // src_np + src_decomp = {i: list(range(i * elements_per_rank_src, + (i + 1) * elements_per_rank_src)) + for i in range(src_np)} + + elements_per_rank_trg = total_elements // trg_np + trg_decomp = {i: list(range(i * elements_per_rank_trg, + (i + 1) * elements_per_rank_trg)) + for i in range(trg_np)} + + # Adjust the last rank to include any remaining elements + src_decomp[src_np - 1].extend(range(src_np * elements_per_rank_src, + total_elements)) + trg_decomp[trg_np - 1].extend(range(trg_np * elements_per_rank_trg, + total_elements)) + + # Testing volume decomps + # Vol1: even elements, Vol2: odd elements + vol1_elements = [i for i in range(total_elements) if i % 2 == 0] + vol2_elements = [i for i in range(total_elements) if i % 2 != 0] + + src_vol_decomp = {} + trg_vol_decomp = {} + + # Test vol decomps + for i in range(src_np): + src_vol_decomp[PartID(volume_tag="vol1", rank=i)] = \ + [el for el in src_decomp[i] if el in vol1_elements] + src_vol_decomp[PartID(volume_tag="vol2", rank=i)] = \ + [el for el in src_decomp[i] if el in vol2_elements] + + for i in range(trg_np): + trg_vol_decomp[PartID(volume_tag="vol1", rank=i)] = \ + [el for el in trg_decomp[i] if el in vol1_elements] + trg_vol_decomp[PartID(volume_tag="vol2", rank=i)] = \ + [el for el in trg_decomp[i] if el in vol2_elements] + + from mirgecom.simutil import multivolume_interdecomposition_overlap + # Compute the multivolume interdecomp overlaps + mv_idx = multivolume_interdecomposition_overlap( + trg_decomp, src_decomp, trg_vol_decomp, src_vol_decomp + ) + + # Perform some basic checks on the returned data structure + # The num trg ranks in the overlap should match the input + assert set(mv_idx.keys()) == set(trg_decomp.keys()) + + for _trg_rank, src_rank_mappings in mv_idx.items(): + for src_rank, partid_mappings in src_rank_mappings.items(): + for partid, element_mappings in partid_mappings.items(): + for trg_local_idx, src_local_idx in element_mappings.items(): + # match element ids for the trg and src at resp index + assert trg_vol_decomp[partid][trg_local_idx] == \ + src_vol_decomp[PartID(volume_tag=partid.volume_tag, + rank=src_rank)][src_local_idx] + + +def _generate_decompositions(total_elements, num_ranks, pattern="chunked"): + """Generate testing decomp.""" + elements_per_rank = total_elements // num_ranks + if pattern == "chunked": + decomp = {i: list(range(i * elements_per_rank, + (i + 1) * elements_per_rank)) + for i in range(num_ranks)} + # Toss remaining els into last rank + decomp[num_ranks - 1].extend(range(num_ranks * elements_per_rank, + total_elements)) + elif pattern == "strided": + decomp = {i: list(range(i, total_elements, num_ranks)) + for i in range(num_ranks)} + elif pattern == "random": + all_elements = list(range(total_elements)) + np.random.shuffle(all_elements) + decomp = {i: all_elements[i * elements_per_rank: (i + 1) * elements_per_rank] + for i in range(num_ranks)} + decomp[num_ranks - 1].extend(all_elements[num_ranks * elements_per_rank:]) + return decomp + + +@pytest.mark.parametrize("decomp_pattern", ["chunked", "strided", "random"]) +@pytest.mark.parametrize("vol_pattern", ["front_back", "random_split"]) +@pytest.mark.parametrize("src_trg_ranks", [(3, 4), (4, 4), (5, 4), (1, 4), (4, 1)]) +def test_multivolume_interdecomp_overlap(decomp_pattern, vol_pattern, src_trg_ranks): + """Test the multivolume_interdecomp_overlap.""" + total_elements = 100 + src_np, trg_np = src_trg_ranks + + # Generate global decomps + src_decomp = _generate_decompositions(total_elements, src_np, + pattern=decomp_pattern) + trg_decomp = _generate_decompositions(total_elements, trg_np, + pattern=decomp_pattern) + + # Volume splitting + if vol_pattern == "front_back": + mid_point = total_elements // 2 + vol1_elements = list(range(mid_point)) + vol2_elements = list(range(mid_point, total_elements)) + elif vol_pattern == "random_split": + all_elements = list(range(total_elements)) + np.random.shuffle(all_elements) + mid_point = total_elements // 2 + vol1_elements = all_elements[:mid_point] + vol2_elements = all_elements[mid_point:] + + src_vol_decomp = {} + trg_vol_decomp = {} + + # Form the multivol decomps + for i in range(src_np): + src_vol_decomp[PartID(volume_tag="vol1", rank=i)] = \ + [el for el in src_decomp[i] if el in vol1_elements] + src_vol_decomp[PartID(volume_tag="vol2", rank=i)] = \ + [el for el in src_decomp[i] if el in vol2_elements] + + for i in range(trg_np): + trg_vol_decomp[PartID(volume_tag="vol1", rank=i)] = \ + [el for el in trg_decomp[i] if el in vol1_elements] + trg_vol_decomp[PartID(volume_tag="vol2", rank=i)] = \ + [el for el in trg_decomp[i] if el in vol2_elements] + + # Testing the overlap utility + from mirgecom.simutil import multivolume_interdecomposition_overlap + mv_idx = multivolume_interdecomposition_overlap( + trg_decomp, src_decomp, trg_vol_decomp, src_vol_decomp + ) + + assert set(mv_idx.keys()) == set(trg_decomp.keys()) + + for _trg_rank, src_rank_mappings in mv_idx.items(): + for src_rank, partid_mappings in src_rank_mappings.items(): + for partid, element_mappings in partid_mappings.items(): + for trg_local_idx, src_local_idx in element_mappings.items(): + assert trg_vol_decomp[partid][trg_local_idx] == \ + src_vol_decomp[PartID(volume_tag=partid.volume_tag, + rank=src_rank)][src_local_idx] From 23e9fe7635dfc04cad49cda46889ae125ecc35c4 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Mon, 9 Oct 2023 16:25:58 -0500 Subject: [PATCH 28/55] Path checking on restart files, broken redist util --- mirgecom/restart.py | 183 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 178 insertions(+), 5 deletions(-) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index 49f5ff40c..f192eecef 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -33,6 +33,8 @@ class PathError(RuntimeError): + """Use RuntimeError for filesystem path errors.""" + pass @@ -51,10 +53,11 @@ def write_restart_file(actx, restart_data, filename, comm=None): if rank == 0: import os rst_dir = os.path.dirname(filename) - if os.path.exists(rst_dir) and not os.path.isdir(rst_dir): - raise PathError(f"{rst_dir} exists and is not a directory.") - if not os.path.exists(rst_dir): - os.makedirs(rst_dir, exist_ok=True) + if rst_dir: + if os.path.exists(rst_dir) and not os.path.isdir(rst_dir): + raise PathError(f"{rst_dir} exists and is not a directory.") + elif not os.path.exists(rst_dir): + os.makedirs(rst_dir, exist_ok=True) if comm: comm.barrier() with array_context_for_pickling(actx): @@ -207,7 +210,177 @@ def _recursive_init_with_zeros(sample_item, trg_zeros): mesh_pkl_filename = f"{mesh_filename}_np{trg_nparts}_rank{trg_part}.pkl" with array_context_for_pickling(actx): with open(mesh_pkl_filename, "rb") as pkl_file: - global_nelements, volume_to_local_mesh_data = pickle.load(pkl_file) + global_nelements, volume_to_local_mesh_data = \ + pickle.load(pkl_file) + out_rst_data[mesh_data_item] = volume_to_local_mesh_data + + # Write out the trg_part-specific redistributed pkl restart file + output_filename = f"{output_path}-{trg_part:04d}.pkl" + with array_context_for_pickling(actx): + with open(output_filename, "wb") as f: + pickle.dump(out_rst_data, f) + + return + + +def redistribute_multivolume_restart_data( + actx, comm, source_multivol_decomp_map_file, input_path, + target_multivol_decomp_map_file, output_path, mesh_filename): + """Redistribute a multi-volume restart dataset M-to-N.""" + from meshmode.dof_array import DOFArray + from dataclasses import is_dataclass, asdict + from mirgecom.simutil import ( + invert_decomp, + interdecomposition_overlap, + copy_mapped_dof_array_data + ) + from mpi4py.util import pkl5 + + def _create_zeros_like_dofarray(actx, nel, ary): + # Get nnodes from the shape of the sample DOFArray + _, nnodes = ary[0].shape + zeros_array = actx.zeros(shape=(nel, nnodes), dtype=ary[0].dtype) + return DOFArray(actx, (zeros_array,)) + + # Traverse the restart_data and copy data from the src + # into the target DOFArrays in-place, so that the trg data + # is persistent across multiple calls with different src data. + def _recursive_map_and_copy(trg_item, src_item, elem_map): + """Recursively map and copy DOFArrays from src_item.""" + if trg_item is None: + print(f"{src_item=}") + raise ValueError("trg_item is None, but src_item is not.") + if src_item is None: + print(f"{trg_item=}") + raise ValueError("src_item is None, but trg_item is not.") + # print(f"{trg_item=}") + # print(f"{src_item=}") + if isinstance(src_item, DOFArray): + if trg_item is None: + raise ValueError("No corresponding target DOFArray found.") + return copy_mapped_dof_array_data(trg_item, src_item, elem_map) + elif isinstance(src_item, dict): + return {k: _recursive_map_and_copy(trg_item.get(k, None), v, elem_map) + for k, v in src_item.items()} + elif isinstance(src_item, (list, tuple)): + return type(src_item)(_recursive_map_and_copy(t, v, elem_map) + for t, v in zip(trg_item, src_item)) + elif is_dataclass(src_item): + trg_dict = asdict(trg_item) + return type(src_item)(**{ + k: _recursive_map_and_copy(trg_dict.get(k, None), v, + elem_map) + for k, v in asdict(src_item).items()}) + else: + return src_item # dupe non-dof data outright + + # Creates a restart data set w/zeros of (trg=part-specific) size for + # all the DOFArray data. To be updated in-place with data from each + # src part. + def _recursive_init_with_zeros(sample_item, trg_zeros): + """Recursively initialize data structures with zeros or original data.""" + black_list = ["volume_to_local_mesh_data", "mesh"] + if isinstance(sample_item, DOFArray): + return 1. * trg_zeros + elif isinstance(sample_item, dict): + return {k: _recursive_init_with_zeros(v, trg_zeros) + for k, v in sample_item.items() if k not in black_list} + elif isinstance(sample_item, (list, tuple)): + return type(sample_item)(_recursive_init_with_zeros(v, trg_zeros) + for v in sample_item) + elif is_dataclass(sample_item): + return type(sample_item)(**{k: _recursive_init_with_zeros(v, + trg_zeros) + for k, v in asdict(sample_item).items()}) + else: + return sample_item + + def _ensure_unique_nelems(mesh_data_dict): + seen_nelems = set() + + for volid, mesh_data in mesh_data_dict.items(): + if mesh_data.nelements in seen_nelems: + raise ValueError(f"Multiple volumes {volid} found with same " + "number of elements.") + seen_nelems.add(mesh_data.nelem) + + comm_wrapper = pkl5.Intracomm(comm) + my_rank = comm_wrapper.Get_rank() + + with open(source_multivol_decomp_map_file, "rb") as pkl_file: + src_dcmp = pickle.load(pkl_file) + with open(target_multivol_decomp_map_file, "rb") as pkl_file: + trg_dcmp = pickle.load(pkl_file) + + trg_parts = invert_decomp(trg_dcmp) + trg_nparts = len(trg_parts) + + writer_color = 1 if my_rank < trg_nparts else 0 + writer_comm = comm_wrapper.Split(writer_color, my_rank) + writer_comm_wrapper = pkl5.Intracomm(writer_comm) + + if writer_color: + writer_nprocs = writer_comm_wrapper.Get_size() + writer_rank = writer_comm_wrapper.Get_rank() + nparts_per_writer = int(writer_nprocs / trg_nparts) + nleftover = trg_nparts - (nparts_per_writer * writer_nprocs) + nparts_this_writer = nparts_per_writer + (1 if writer_rank + < nleftover else 0) + my_starting_rank = nparts_per_writer * writer_rank + my_starting_rank = my_starting_rank + (writer_rank if writer_rank + < nleftover else nleftover) + my_ending_rank = my_starting_rank + nparts_this_writer - 1 + parts_to_write = list(range(my_starting_rank, my_ending_rank+1)) + xdo = interdecomposition_overlap(trg_dcmp, src_dcmp, + return_parts=parts_to_write) + + # Read one source restart file to get the 'order' and a sample DOFArray + mesh_data_item = "volume_to_local_mesh_data" + sample_restart_file = f"{input_path}-{writer_rank:04d}.pkl" + with array_context_for_pickling(actx): + with open(sample_restart_file, "rb") as f: + sample_data = pickle.load(f) + if "mesh" in sample_data: + mesh_data_item = "mesh" # check mesh data return type instead + + sample_dof_array = \ + next(val for key, val in + sample_data.items() if isinstance(val, DOFArray)) + + for trg_part, olaps in xdo.items(): + # Number of elements in each target partition + trg_nel = len(trg_parts[trg_part]) + + # Create representative DOFArray with zeros using sample for order + # But with nelem = trg_nel + trg_zeros = _create_zeros_like_dofarray( + actx, trg_nel, sample_dof_array) + # Use trg_zeros to reset all dof arrays in the out_rst_data to + # the current (trg_part) size made of zeros. + with array_context_for_pickling(actx): + out_rst_data = _recursive_init_with_zeros(sample_data, trg_zeros) + + # Read and Map DOFArrays from source to target + for src_part, elem_map in olaps.items(): + # elem_map={trg-local-index : src-local-index} for trg,src parts + src_restart_file = f"{input_path}-{src_part:04d}.pkl" + with array_context_for_pickling(actx): + with open(src_restart_file, "rb") as f: + src_rst_data = pickle.load(f) + vol_to_src_mesh_data = \ + src_rst_data.pop("volume_to_local_mesh_data", None) + _ensure_unique_nelems(vol_to_src_mesh_data) + src_rst_data.pop("mesh", None) + with array_context_for_pickling(actx): + out_rst_data = _recursive_map_and_copy( + out_rst_data, src_rst_data, elem_map) + + # Read new mesh data and stack it in the restart file + mesh_pkl_filename = f"{mesh_filename}_np{trg_nparts}_rank{trg_part}.pkl" + with array_context_for_pickling(actx): + with open(mesh_pkl_filename, "rb") as pkl_file: + global_nelements, volume_to_local_mesh_data = \ + pickle.load(pkl_file) out_rst_data[mesh_data_item] = volume_to_local_mesh_data # Write out the trg_part-specific redistributed pkl restart file From 3ad363af592b2d0b1b4e509d4fcf800d29ed76d6 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 10 Oct 2023 02:42:25 -0500 Subject: [PATCH 29/55] Add m-to-n api for multi-volume datasets. --- mirgecom/restart.py | 327 ++++++++++++++++++++++++++------------------ 1 file changed, 193 insertions(+), 134 deletions(-) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index f192eecef..ba2a8e048 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -29,7 +29,16 @@ """ import pickle -from meshmode.dof_array import array_context_for_pickling +from meshmode.dof_array import array_context_for_pickling, DOFArray +from grudge.discretization import PartID +from dataclasses import is_dataclass, asdict +from collections import defaultdict +from mirgecom.simutil import ( + invert_decomp, + interdecomposition_overlap, + multivolume_interdecomposition_overlap, + copy_mapped_dof_array_data +) class PathError(RuntimeError): @@ -69,13 +78,6 @@ def redistribute_restart_data(actx, comm, source_decomp_map_file, input_path, target_decomp_map_file, output_path, mesh_filename): """Redistribute a restart dataset M-to-N.""" - from meshmode.dof_array import DOFArray - from dataclasses import is_dataclass, asdict - from mirgecom.simutil import ( - invert_decomp, - interdecomposition_overlap, - copy_mapped_dof_array_data - ) from mpi4py.util import pkl5 def _create_zeros_like_dofarray(actx, nel, ary): @@ -223,97 +225,185 @@ def _recursive_init_with_zeros(sample_item, trg_zeros): return -def redistribute_multivolume_restart_data( - actx, comm, source_multivol_decomp_map_file, input_path, - target_multivol_decomp_map_file, output_path, mesh_filename): - """Redistribute a multi-volume restart dataset M-to-N.""" - from meshmode.dof_array import DOFArray - from dataclasses import is_dataclass, asdict - from mirgecom.simutil import ( - invert_decomp, - interdecomposition_overlap, - copy_mapped_dof_array_data - ) - from mpi4py.util import pkl5 - - def _create_zeros_like_dofarray(actx, nel, ary): - # Get nnodes from the shape of the sample DOFArray - _, nnodes = ary[0].shape - zeros_array = actx.zeros(shape=(nel, nnodes), dtype=ary[0].dtype) +def _find_rank_with_all_volumes(multivol_decomp_map): + """Find a rank that has data for all volumes.""" + # Collect all volume tags from the PartIDs + all_volumes = \ + {partid.volume_tag for partid in multivol_decomp_map.keys()} + + # Track which ranks have seen which volumes + ranks_seen_volumes = defaultdict(set) + + for partid, elements in multivol_decomp_map.items(): + if elements: # non-empty means this rank has data for this volume + ranks_seen_volumes[partid.rank].add(partid.volume_tag) + + # Now, find a rank that has seen all volumes + for rank, seen_volumes in ranks_seen_volumes.items(): + if seen_volumes == all_volumes: + return rank + + raise ValueError("No rank found with data for all volumes.") + + +# Traverse the restart_data and copy data from the src +# into the target DOFArrays in-place, so that the trg data +# is persistent across multiple calls with different src data. +def _recursive_map_and_copy(trg_item, src_item, trg_partid_to_index_map, + src_volume_sizes): + """Recursively map and copy DOFArrays from src_item.""" + if trg_item is None: + print(f"{src_item=}") + raise ValueError("trg_item is None, but src_item is not.") + if src_item is None: + print(f"{trg_item=}") + raise ValueError("src_item is None, but trg_item is not.") + trg_rank = next(iter(trg_partid_to_index_map)).rank + + # print(f"{trg_item=}") + # print(f"{src_item=}") + src_nel, src_nnodes = src_item[0].shape + volume_tag = next((vol_tag for vol_tag, size in src_volume_sizes.items() + if size == src_nel), None) + target_partid = PartID(volume_tag=volume_tag, rank=trg_rank) + elem_map = trg_partid_to_index_map[target_partid] + + if isinstance(src_item, DOFArray): + if trg_item is None: + raise ValueError("No corresponding target DOFArray found.") + return copy_mapped_dof_array_data(trg_item, src_item, elem_map) + elif isinstance(src_item, dict): + return {k: _recursive_map_and_copy( + trg_item.get(k, None), v, trg_partid_to_index_map, + src_volume_sizes) for k, v in src_item.items()} + elif isinstance(src_item, (list, tuple)): + return type(src_item)(_recursive_map_and_copy( + t, v, trg_partid_to_index_map, src_volume_sizes) + for t, v in zip(trg_item, src_item)) + elif is_dataclass(src_item): + trg_dict = asdict(trg_item) + return type(src_item)(**{ + k: _recursive_map_and_copy( + trg_dict.get(k, None), v, trg_partid_to_index_map, + src_volume_sizes) for k, v in asdict(src_item).items()}) + else: + return src_item # dupe non-dof data outright + + +def _ensure_unique_nelems(mesh_data_dict): + seen_nelems = set() + + for volid, mesh_data in mesh_data_dict.items(): + if mesh_data.nelements in seen_nelems: + raise ValueError(f"Multiple volumes {volid} found with same " + "number of elements.") + seen_nelems.add(mesh_data.nelem) + + +def _get_volume_sizes_on_each_rank(multivol_decomp_map): + volume_sizes = defaultdict(int) + for partid, elements in multivol_decomp_map.items(): + volume_sizes[(partid.rank, partid.volume_tag)] = len(elements) + return volume_sizes + + +def _recursive_resize_reinit_with_zeros(actx, sample_item, target_volume_sizes, + sample_volume_sizes): + """Recursively initialize data structures with zeros or original data.""" + if isinstance(sample_item, DOFArray): + sample_nel, sample_nnodes = sample_item[0].shape + volume_tag = next((vol_tag for vol_tag, size in sample_volume_sizes.items() + if size == sample_nel), None) + trg_nel = target_volume_sizes[volume_tag] + _, nnodes = sample_item[0].shape + zeros_array = actx.zeros(shape=(trg_nel, sample_nnodes), + dtype=sample_item[0].dtype) return DOFArray(actx, (zeros_array,)) + elif isinstance(sample_item, dict): + return {k: _recursive_resize_reinit_with_zeros(actx, v, target_volume_sizes) + for k, v in sample_item.items()} + elif isinstance(sample_item, (list, tuple)): + return type(sample_item)(_recursive_resize_reinit_with_zeros(actx, v, + target_volume_sizes) + for v in sample_item) + elif is_dataclass(sample_item): + return type(sample_item)(**{k: _recursive_resize_reinit_with_zeros( + actx, v, target_volume_sizes) for k, v in asdict(sample_item).items()}) + else: + return sample_item # retain non-dof data outright - # Traverse the restart_data and copy data from the src - # into the target DOFArrays in-place, so that the trg data - # is persistent across multiple calls with different src data. - def _recursive_map_and_copy(trg_item, src_item, elem_map): - """Recursively map and copy DOFArrays from src_item.""" - if trg_item is None: - print(f"{src_item=}") - raise ValueError("trg_item is None, but src_item is not.") - if src_item is None: - print(f"{trg_item=}") - raise ValueError("src_item is None, but trg_item is not.") - # print(f"{trg_item=}") - # print(f"{src_item=}") - if isinstance(src_item, DOFArray): - if trg_item is None: - raise ValueError("No corresponding target DOFArray found.") - return copy_mapped_dof_array_data(trg_item, src_item, elem_map) - elif isinstance(src_item, dict): - return {k: _recursive_map_and_copy(trg_item.get(k, None), v, elem_map) - for k, v in src_item.items()} - elif isinstance(src_item, (list, tuple)): - return type(src_item)(_recursive_map_and_copy(t, v, elem_map) - for t, v in zip(trg_item, src_item)) - elif is_dataclass(src_item): - trg_dict = asdict(trg_item) - return type(src_item)(**{ - k: _recursive_map_and_copy(trg_dict.get(k, None), v, - elem_map) - for k, v in asdict(src_item).items()}) - else: - return src_item # dupe non-dof data outright - # Creates a restart data set w/zeros of (trg=part-specific) size for - # all the DOFArray data. To be updated in-place with data from each - # src part. - def _recursive_init_with_zeros(sample_item, trg_zeros): - """Recursively initialize data structures with zeros or original data.""" - black_list = ["volume_to_local_mesh_data", "mesh"] - if isinstance(sample_item, DOFArray): - return 1. * trg_zeros - elif isinstance(sample_item, dict): - return {k: _recursive_init_with_zeros(v, trg_zeros) - for k, v in sample_item.items() if k not in black_list} - elif isinstance(sample_item, (list, tuple)): - return type(sample_item)(_recursive_init_with_zeros(v, trg_zeros) - for v in sample_item) - elif is_dataclass(sample_item): - return type(sample_item)(**{k: _recursive_init_with_zeros(v, - trg_zeros) - for k, v in asdict(sample_item).items()}) - else: - return sample_item +def _get_volume_sizes_for_rank(target_rank, multivol_decomp_map): + volume_sizes = {} + + for partid, elements in multivol_decomp_map.items(): + if partid.rank == target_rank: + volume_sizes[partid.volume_tag] = len(elements) + + return volume_sizes - def _ensure_unique_nelems(mesh_data_dict): - seen_nelems = set() - for volid, mesh_data in mesh_data_dict.items(): - if mesh_data.nelements in seen_nelems: - raise ValueError(f"Multiple volumes {volid} found with same " - "number of elements.") - seen_nelems.add(mesh_data.nelem) +def _get_restart_data_for_target_rank( + actx, trg_rank, sample_rst_data, sample_vol_sizes, src_overlaps, + target_multivol_decomp_map, source_multivol_decomp_map, + input_path): + trg_vol_sizes = _get_volume_sizes_for_rank( + trg_rank, target_multivol_decomp_map) + + with array_context_for_pickling(actx): + out_rst_data = _recursive_resize_reinit_with_zeros( + actx, sample_rst_data, trg_vol_sizes, sample_vol_sizes) + + # Read and Map DOFArrays from source to target + for src_rank, trg_partid_to_idx_map in src_overlaps.items(): + src_vol_sizes = _get_volume_sizes_for_rank( + src_rank, source_multivol_decomp_map) + + src_restart_file = f"{input_path}-{src_rank:04d}.pkl" + with array_context_for_pickling(actx): + with open(src_restart_file, "rb") as f: + src_rst_data = pickle.load(f) + mesh_data = src_rst_data.pop("volume_to_local_mesh_data", None) + _ensure_unique_nelems(mesh_data) + + with array_context_for_pickling(actx): + out_rst_data = _recursive_map_and_copy( + out_rst_data, src_rst_data, trg_partid_to_idx_map, + src_vol_sizes) + + return out_rst_data + + +def redistribute_multivolume_restart_data_new( + actx, comm, source_idecomp_map, target_idecomp_map, + source_multivol_decomp_map, target_multivol_decomp_map, + src_input_path, output_path, mesh_filename): + """Redist a mv ds.""" + from mpi4py.util import pkl5 comm_wrapper = pkl5.Intracomm(comm) my_rank = comm_wrapper.Get_rank() - with open(source_multivol_decomp_map_file, "rb") as pkl_file: - src_dcmp = pickle.load(pkl_file) - with open(target_multivol_decomp_map_file, "rb") as pkl_file: - trg_dcmp = pickle.load(pkl_file) + # Identify a source rank with data for all volumes + sample_rank = _find_rank_with_all_volumes(source_multivol_decomp_map) + if sample_rank is None: + raise ValueError("No source rank found with data for all volumes.") - trg_parts = invert_decomp(trg_dcmp) - trg_nparts = len(trg_parts) + mesh_data_item = "volume_to_local_mesh_data" + sample_restart_file = f"{src_input_path}-{sample_rank:04d}.pkl" + with array_context_for_pickling(actx): + with open(sample_restart_file, "rb") as f: + sample_rst_data = pickle.load(f) + if "mesh" in sample_rst_data: + mesh_data_item = "mesh" # check mesh data return type instead + vol_to_sample_mesh_data = \ + sample_rst_data.pop(mesh_data_item, None) + _ensure_unique_nelems(vol_to_sample_mesh_data) + # sample_vol_sizes, determine from mesh? + sample_vol_sizes = _get_volume_sizes_for_rank(sample_rank, + source_multivol_decomp_map) + + trg_nparts = len(target_idecomp_map) writer_color = 1 if my_rank < trg_nparts else 0 writer_comm = comm_wrapper.Split(writer_color, my_rank) @@ -331,60 +421,29 @@ def _ensure_unique_nelems(mesh_data_dict): < nleftover else nleftover) my_ending_rank = my_starting_rank + nparts_this_writer - 1 parts_to_write = list(range(my_starting_rank, my_ending_rank+1)) - xdo = interdecomposition_overlap(trg_dcmp, src_dcmp, - return_parts=parts_to_write) + xdo = multivolume_interdecomposition_overlap(target_idecomp_map, + source_idecomp_map, + target_multivol_decomp_map, + source_multivol_decomp_map, + return_ranks=parts_to_write) - # Read one source restart file to get the 'order' and a sample DOFArray - mesh_data_item = "volume_to_local_mesh_data" - sample_restart_file = f"{input_path}-{writer_rank:04d}.pkl" - with array_context_for_pickling(actx): - with open(sample_restart_file, "rb") as f: - sample_data = pickle.load(f) - if "mesh" in sample_data: - mesh_data_item = "mesh" # check mesh data return type instead + for trg_rank, olaps in xdo.items(): - sample_dof_array = \ - next(val for key, val in - sample_data.items() if isinstance(val, DOFArray)) - - for trg_part, olaps in xdo.items(): - # Number of elements in each target partition - trg_nel = len(trg_parts[trg_part]) - - # Create representative DOFArray with zeros using sample for order - # But with nelem = trg_nel - trg_zeros = _create_zeros_like_dofarray( - actx, trg_nel, sample_dof_array) - # Use trg_zeros to reset all dof arrays in the out_rst_data to - # the current (trg_part) size made of zeros. - with array_context_for_pickling(actx): - out_rst_data = _recursive_init_with_zeros(sample_data, trg_zeros) - - # Read and Map DOFArrays from source to target - for src_part, elem_map in olaps.items(): - # elem_map={trg-local-index : src-local-index} for trg,src parts - src_restart_file = f"{input_path}-{src_part:04d}.pkl" - with array_context_for_pickling(actx): - with open(src_restart_file, "rb") as f: - src_rst_data = pickle.load(f) - vol_to_src_mesh_data = \ - src_rst_data.pop("volume_to_local_mesh_data", None) - _ensure_unique_nelems(vol_to_src_mesh_data) - src_rst_data.pop("mesh", None) - with array_context_for_pickling(actx): - out_rst_data = _recursive_map_and_copy( - out_rst_data, src_rst_data, elem_map) + out_rst_data = _get_restart_data_for_target_rank( + actx, trg_rank, sample_rst_data, sample_vol_sizes, + olaps, target_multivol_decomp_map, source_multivol_decomp_map, + src_input_path) # Read new mesh data and stack it in the restart file - mesh_pkl_filename = f"{mesh_filename}_np{trg_nparts}_rank{trg_part}.pkl" + mesh_pkl_filename = f"{mesh_filename}_np{trg_nparts}_rank{trg_rank}.pkl" with array_context_for_pickling(actx): with open(mesh_pkl_filename, "rb") as pkl_file: - global_nelements, volume_to_local_mesh_data = \ + global_nelements, mesh_data = \ pickle.load(pkl_file) - out_rst_data[mesh_data_item] = volume_to_local_mesh_data + out_rst_data[mesh_data_item] = mesh_data # Write out the trg_part-specific redistributed pkl restart file - output_filename = f"{output_path}-{trg_part:04d}.pkl" + output_filename = f"{output_path}-{trg_rank:04d}.pkl" with array_context_for_pickling(actx): with open(output_filename, "wb") as f: pickle.dump(out_rst_data, f) From 3f4cef579d0bbfc1a18b3fb29bdd6c67caad3f57 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 10 Oct 2023 02:43:47 -0500 Subject: [PATCH 30/55] Sharpen doc slightly --- mirgecom/simutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index ebf469989..b0588c300 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -1631,7 +1631,7 @@ def multivolume_interdecomposition_overlap(target_idecomp, source_idecomp, { target_rank: { source_rank: { - PartID: { + (targ)PartID: { target_local_index: source_local_index } } From 6473120479edf2bd516c8ad33ca4bee36381db6a Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 10 Oct 2023 03:17:43 -0500 Subject: [PATCH 31/55] Sharpen some docs --- mirgecom/restart.py | 80 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index ba2a8e048..40fec070a 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -251,7 +251,25 @@ def _find_rank_with_all_volumes(multivol_decomp_map): # is persistent across multiple calls with different src data. def _recursive_map_and_copy(trg_item, src_item, trg_partid_to_index_map, src_volume_sizes): - """Recursively map and copy DOFArrays from src_item.""" + """ + Recursively map and copy DOFArrays from the source item to the target item. + + Parameters + ---------- + trg_item: object + The target item where data will be mapped and copied. + src_item: object + The source item from which data will be copied. + trg_partid_to_index_map: dict + A mapping from PartID to index for the target. + src_volume_sizes: dict + Dictionary of volume sizes for the source. + + Returns + ------- + object: + The target item after mapping and copying the data from the source item. + """ if trg_item is None: print(f"{src_item=}") raise ValueError("trg_item is None, but src_item is not.") @@ -292,7 +310,6 @@ def _recursive_map_and_copy(trg_item, src_item, trg_partid_to_index_map, def _ensure_unique_nelems(mesh_data_dict): seen_nelems = set() - for volid, mesh_data in mesh_data_dict.items(): if mesh_data.nelements in seen_nelems: raise ValueError(f"Multiple volumes {volid} found with same " @@ -309,7 +326,28 @@ def _get_volume_sizes_on_each_rank(multivol_decomp_map): def _recursive_resize_reinit_with_zeros(actx, sample_item, target_volume_sizes, sample_volume_sizes): - """Recursively initialize data structures with zeros or original data.""" + """ + Recursively initialize a composite data structure based on a sample. + + DOFArray data items are initialized with zeros of the appropriate + target partid size. Non DOFArray items are copied from original sample. + + Parameters + ---------- + actx: :class:`arraycontext.ArrayContext` + The array context used for operations. + sample_item: object + A sample item to base the initialization on. + target_volume_sizes: dict + Target volume sizes. + sample_volume_sizes: dict + Sample volume sizes. + + Returns + ------- + object: + Initialized data structure. + """ if isinstance(sample_item, DOFArray): sample_nel, sample_nnodes = sample_item[0].shape volume_tag = next((vol_tag for vol_tag, size in sample_volume_sizes.items() @@ -378,7 +416,41 @@ def redistribute_multivolume_restart_data_new( actx, comm, source_idecomp_map, target_idecomp_map, source_multivol_decomp_map, target_multivol_decomp_map, src_input_path, output_path, mesh_filename): - """Redist a mv ds.""" + """ + Redistribute (m-to-n) multi-volume restart data. + + This function takes in src(m) and trg(n) decomps for multi-volume datasets. + It then redistributes the restart data from src to match the trg decomposition. + + Parameters + ---------- + actx: :class:`arraycontext.ArrayContext` + The array context used for operations + comm: + Am MPI communicator object + source_idecomp_map: dict + Decomposition map of the source distribution without volume tags. + target_idecomp_map: dict + Decomposition map of the target distribution without volume tags. + source_multivol_decomp_map: dict + Decomposition map of the source with volume tags. It maps from src `PartID` + objects to lists of elements. + target_multivol_decomp_map: dict + Decomposition map of the target with volume tags. It maps from trg 'PartID' + objects to lists of elements. + src_input_path: str + Path to the source restart data files. + output_path: str + Path to save the redistributed restart data files. + mesh_filename: str + Base filename of the mesh data for the restart data + + Returns + ------- + None + This function doesn't return any value but writes the redistributed + restart data to the specified `output_path`. + """ from mpi4py.util import pkl5 comm_wrapper = pkl5.Intracomm(comm) From fd6225a6f091a5979ec7bc642a3304410afa0b2b Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 10 Oct 2023 04:23:36 -0500 Subject: [PATCH 32/55] Extend redist to multi-volume datasets. --- bin/redist.py | 44 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/bin/redist.py b/bin/redist.py index 8788683aa..1244bc7f4 100644 --- a/bin/redist.py +++ b/bin/redist.py @@ -47,7 +47,8 @@ from mirgecom.simutil import ( ApplicationOptionsError, - distribute_mesh_pkl + distribute_mesh_pkl, + invert_decomp ) from mirgecom.mpi import mpi_entry_point @@ -278,24 +279,32 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): source_mesh_data = get_mesh_data() from meshmode.mesh import Mesh if isinstance(source_mesh_data, Mesh): + multivolume_dataset = False source_mesh = source_mesh_data tag_to_elements = None volume_to_tags = None elif isinstance(source_mesh_data, tuple): source_mesh, tag_to_elements, volume_to_tags = source_mesh_data + multivolume_dataset = True else: raise TypeError("Unexpected result from get_mesh_data") + comm.bcast(multivolume_dataset) rank_per_element = my_partitioner(source_mesh, tag_to_elements, mdist) with open(source_decomp_map_file, "wb") as pkl_file: pickle.dump(rank_per_element, pkl_file) print("Done generating source decomp.") + else: + multivolume_dataset = comm.bcast(None) comm.Barrier() if rank == 0: - print(f"Partitioning mesh to {ndist} parts, writing to {mesh_filename}...") + meshtype = " multi-volume " if multivolume_dataset else "" + print(f"Partitioning {meshtype} mesh to {ndist} parts, " + f"writing to {mesh_filename}...") # This bit creates the N-parted mesh pkl files and partition table + # Should only do this if the n decomp map is not found in the output path. distribute_mesh_pkl( comm, get_mesh_data, filename=mesh_filename, num_target_ranks=ndist, partition_generator_func=part_func, logmgr=logmgr) @@ -306,10 +315,33 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): print("Done partitioning target mesh, mesh pkl files written.") print(f"Generating the restart data for {ndist} parts...") - target_decomp_map_file = mesh_filename + f"_decomp_np{ndist}.pkl" - from mirgecom.restart import redistribute_restart_data - redistribute_restart_data(actx, comm, source_decomp_map_file, input_path, - target_decomp_map_file, output_path, mesh_filename) + if multivolume_dataset: + target_decomp_map_file = mesh_filename + f"_decomp_np{ndist}.pkl" + target_multivol_decomp_map_file = \ + mesh_filename + f"_multivol_idecomp_np{ndist}.pkl" + with open(target_decomp_map_file, "rb") as pkl_file: + trg_dcmp = pickle.load(pkl_file) + trg_idcmp = invert_decomp(trg_dcmp) + with open(target_multivol_decomp_map_file, "rb") as pkl_file: + trg_mv_dcmp = pickle.load(pkl_file) + source_decomp_map_file = input_path + f"_decomp_np{mdist}.pkl" + source_multivol_decomp_map_file = \ + input_path + f"_multivol_idcomp_np{mdist}.pkl" + with open(source_decomp_map_file, "rb") as pkl_file: + src_dcmp = pickle.load(pkl_file) + src_idcmp = invert_decomp(src_dcmp) + with open(source_multivol_decomp_map_file, "rb") as pkl_file: + src_mv_dcmp = pickle.load(pkl_file) + from mirgecom.restart import redistribute_multivolume_restart_data + redistribute_multivolume_restart_data( + actx, comm, src_idcmp, trg_idcmp, + src_mv_dcmp, trg_mv_dcmp, input_path, + output_path, mesh_filename) + else: + target_decomp_map_file = mesh_filename + f"_decomp_np{ndist}.pkl" + from mirgecom.restart import redistribute_restart_data + redistribute_restart_data(actx, comm, source_decomp_map_file, input_path, + target_decomp_map_file, output_path, mesh_filename) if rank == 0: print("Done generating restart data.") From c35c86dc305a3a268d9f8930bc71961cc38ea7f2 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 10 Oct 2023 04:25:03 -0500 Subject: [PATCH 33/55] Unnewify function name --- mirgecom/restart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index 40fec070a..9a5fa0055 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -412,7 +412,7 @@ def _get_restart_data_for_target_rank( return out_rst_data -def redistribute_multivolume_restart_data_new( +def redistribute_multivolume_restart_data( actx, comm, source_idecomp_map, target_idecomp_map, source_multivol_decomp_map, target_multivol_decomp_map, src_input_path, output_path, mesh_filename): From 878412a206b523246a8d1f93de9e26a0f1b3f709 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 10 Oct 2023 19:09:38 -0500 Subject: [PATCH 34/55] Cleanup after refac --- examples/ablation-workshop.py | 2 +- mirgecom/restart.py | 14 ++++++++------ mirgecom/simutil.py | 9 +++++---- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/examples/ablation-workshop.py b/examples/ablation-workshop.py index 3e8d46920..a06d7536a 100644 --- a/examples/ablation-workshop.py +++ b/examples/ablation-workshop.py @@ -27,7 +27,7 @@ import numpy as np import scipy # type: ignore[import] from scipy.interpolate import CubicSpline # type: ignore[import] - +from warnings import warn from meshmode.dof_array import DOFArray from meshmode.discretization.connection import FACE_RESTR_ALL diff --git a/mirgecom/restart.py b/mirgecom/restart.py index 9a5fa0055..d4576a83a 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -358,15 +358,17 @@ def _recursive_resize_reinit_with_zeros(actx, sample_item, target_volume_sizes, dtype=sample_item[0].dtype) return DOFArray(actx, (zeros_array,)) elif isinstance(sample_item, dict): - return {k: _recursive_resize_reinit_with_zeros(actx, v, target_volume_sizes) - for k, v in sample_item.items()} + return {k: _recursive_resize_reinit_with_zeros( + actx, v, target_volume_sizes, sample_volume_sizes) + for k, v in sample_item.items()} elif isinstance(sample_item, (list, tuple)): - return type(sample_item)(_recursive_resize_reinit_with_zeros(actx, v, - target_volume_sizes) - for v in sample_item) + return type(sample_item)(_recursive_resize_reinit_with_zeros( + actx, v, target_volume_sizes, sample_volume_sizes) + for v in sample_item) elif is_dataclass(sample_item): return type(sample_item)(**{k: _recursive_resize_reinit_with_zeros( - actx, v, target_volume_sizes) for k, v in asdict(sample_item).items()}) + actx, v, target_volume_sizes, sample_volume_sizes) + for k, v in asdict(sample_item).items()}) else: return sample_item # retain non-dof data outright diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index b0588c300..4669b0b57 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -1666,10 +1666,11 @@ def multivolume_interdecomposition_overlap(target_idecomp, source_idecomp, common_elements_set = \ target_elements_set.intersection(source_elements_set) - for trg_el in common_elements_set: - trg_local_idx = target_vol_decomp[targ_partid].index(trg_el) - src_local_idx = source_vol_decomp[src_partid].index(trg_el) - overlap_maps[trg_rank][src_rank][targ_partid][trg_local_idx] = src_local_idx + for trg_el in common_elements_set: + trg_local_idx = target_vol_decomp[targ_partid].index(trg_el) + src_local_idx = source_vol_decomp[src_partid].index(trg_el) + overlap_maps[trg_rank][src_rank][targ_partid][trg_local_idx] = \ + src_local_idx return overlap_maps From 82609e9cc7771723114dfdeb777f25aaaf10fc4f Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 11 Oct 2023 12:11:53 -0500 Subject: [PATCH 35/55] Deep changes to remove complexity, requires input mappings now. --- bin/redist.py | 272 ++++++++++++++++++-------------------------------- 1 file changed, 99 insertions(+), 173 deletions(-) diff --git a/bin/redist.py b/bin/redist.py index 1244bc7f4..0354bad28 100644 --- a/bin/redist.py +++ b/bin/redist.py @@ -27,7 +27,6 @@ import argparse import sys import os -import glob import pickle # from pytools.obj_array import make_obj_array @@ -47,9 +46,8 @@ from mirgecom.simutil import ( ApplicationOptionsError, - distribute_mesh_pkl, - invert_decomp ) + from mirgecom.mpi import mpi_entry_point @@ -75,16 +73,54 @@ class MyRuntimeError(RuntimeError): pass +def _is_mesh_data_multivol(mesh_filename, npart): + mesh_root = mesh_filename + "_mesh" + mvdecomp_filename = mesh_root + f"_multivol_idecomp_np{npart}.pkl" + return os.path.exists(mvdecomp_filename) + + +def _validate_mesh_data_files(mesh_filename, npart, mv=False): + mesh_root = mesh_filename + "_mesh" + part_file_root = mesh_root + f"_np{npart}_rank" + decomp_filename = mesh_root + f"_decomp_np{npart}.pkl" + idecomp_filename = mesh_root + f"_idecomp_np{npart}.pkl" + mvdecomp_filename = mesh_root + f"_multivol_idecomp_np{npart}.pkl" + + data_is_good = True + if not os.path.exists(decomp_filename): + print(f"Failed to find mesh decomp file: {decomp_filename}.") + data_is_good = False + if not os.path.exists(idecomp_filename): + print(f"Failed to find mesh idecomp file: {idecomp_filename}.") + data_is_good = False + if mv: + if not os.path.exists(mvdecomp_filename): + print(f"Failed to find multivol decomp file: {mvdecomp_filename}.") + data_is_good = False + bad_ranks = [] + for r in range(npart): + part_filename = part_file_root + f"{r}.pkl" + if not os.path.exists(part_filename): + bad_ranks.append(r) + data_is_good = False + if len(bad_ranks) > 0: + print(f"Failed to find mesh data for ranks {bad_ranks}.") + if not data_is_good: + raise ApplicationOptionsError( + f"Could not find expected mesh data for {mesh_filename}.") + + @mpi_entry_point def main(actx_class, mesh_source=None, ndist=None, mdist=None, output_path=None, input_path=None, log_path=None, - casename=None, use_1d_part=None, use_wall=False, - restart_file=None): + src_mesh_filename=None, trg_mesh_filename=False): """Redistribute a mirgecom restart dataset.""" - if mesh_source is None: - raise ApplicationOptionsError("Missing mesh source file.") - - mesh_source.strip("'") + from mpi4py import MPI + from mpi4py.util import pkl5 + comm_world = MPI.COMM_WORLD + comm = pkl5.Intracomm(comm_world) + rank = comm.Get_rank() + nprocs = comm.Get_size() if log_path is None: log_path = "log_data" @@ -98,6 +134,19 @@ def main(actx_class, mesh_source=None, ndist=None, mdist=None, if input_path is None: raise ApplicationOptionsError("Input path/filename is required.") + if src_mesh_filename is None: + raise ApplicationOptionsError("Source mesh filename must be specified.") + + if trg_mesh_filename is None: + raise ApplicationOptionsError("Target mesh filename must be specified.") + + # Default to decomp for one part per process + if ndist is None: + ndist = nprocs + + if mdist is None: + raise ApplicationOptionsError("Number of src ranks (m) is unspecified.") + # control log messages logger = logging.getLogger(__name__) logger.propagate = False @@ -117,51 +166,13 @@ def main(actx_class, mesh_source=None, ndist=None, mdist=None, h2.addFilter(f2) logger.addHandler(h2) - from mpi4py import MPI - from mpi4py.util import pkl5 - comm_world = MPI.COMM_WORLD - comm = pkl5.Intracomm(comm_world) - rank = comm.Get_rank() - nprocs = comm.Get_size() - - # Default to decomp for one part per process - if ndist is None: - ndist = nprocs - - if mdist is None: - # Try to detect it. If can't then fail. - if rank == 0: - search_pattern = input_path + "*" - print(f"Searching input path {search_pattern}.") - files = glob.glob(search_pattern) - print(f"Found files: {files}") - xps = ["_decomp_", "_mesh_"] - ffiles = [f for f in files if not any(xc in f for xc in xps)] - mdist = len(ffiles) - if mdist <= 0: - ffiles = [f for f in files if "_decomp_" not in f] - mdist = len(ffiles) - if mdist <= 0: - mdist = len(files) - mdist = comm.bcast(mdist, root=0) - if mdist <= 0: - raise ApplicationOptionsError("Cannot detect number of parts " - "for input data.") - else: - if rank == 0: - print(f"Automatically detected {mdist} input parts.") - - # We need a decomp map for the input data - # If can't find, then generate one. input_data_directory = os.path.dirname(input_path) output_filename = os.path.basename(input_path) - casename = casename or output_filename - casename.strip("'") if os.path.exists(output_path): if not os.path.isdir(output_path): raise ApplicationOptionsError( - "Mesh dist mode requires 'output'" + "Redist mode requires 'output'" " parameter to be a directory for output.") if rank == 0: if not os.path.exists(output_path): @@ -169,35 +180,20 @@ def main(actx_class, mesh_source=None, ndist=None, mdist=None, output_directory = output_path output_path = output_directory + "/" + output_filename - mesh_filename = output_directory + "/" + casename + "_mesh" - if output_path == input_data_directory: + if output_directory == input_data_directory: raise ApplicationOptionsError("Output path must be different than input" " location because of filename collisions.") - decomp_map_file_search_pattern = \ - input_data_directory + f"/*_decomp_np{mdist}.pkl" - input_decomp_map_files = glob.glob(decomp_map_file_search_pattern) - source_decomp_map_file = \ - input_decomp_map_files[0] if input_decomp_map_files else None - - generate_source_decomp = \ - True if source_decomp_map_file is None else False - if source_decomp_map_file is None: - source_decomp_map_file = input_path + f"_decomp_np{mdist}.pkl" - if generate_source_decomp: - print("Unable to find source decomp map, generating from scratch.") - else: - print("Found existing source decomp map.") - print(f"Source decomp map file: {source_decomp_map_file}.") - if rank == 0: - print(f"Redist on {nprocs} procs: {mdist}->{ndist} parts") - print(f"Casename: {casename}") - print(f"Mesh source file: {mesh_source}") + print(f"Redist on {nprocs} procs: {mdist}->{ndist} MPI ranks") + print(f"Source mesh: {src_mesh_filename}") + print(f"Target mesh: {trg_mesh_filename}") + print(f"Input restart data: {input_path}") + print(f"Output restart data: {output_path}") # logging and profiling - logname = log_path + "/" + casename + ".sqlite" + logname = log_path + f"/redist-{mdist}-to-{ndist}.sqlite" if rank == 0: log_dir = os.path.dirname(logname) @@ -239,109 +235,42 @@ def main(actx_class, mesh_source=None, ndist=None, mdist=None, ("memory_usage_hwm.max", "| \t memory hwm: {value:7g} Mb\n")]) - if rank == 0: - print(f"Reading mesh from {mesh_source}.") - print(f"Writing {ndist} mesh pkl files to {output_path}.") - - def get_mesh_data(): - from meshmode.mesh.io import read_gmsh - mesh, tag_to_elements = read_gmsh( - mesh_source, - return_tag_to_elements_map=True) - volume_to_tags = { - "fluid": ["fluid"]} - if use_wall: - volume_to_tags["wall"] = ["wall_insert", "wall_surround"] - else: - from mirgecom.simutil import extract_volumes - mesh, tag_to_elements = extract_volumes( - mesh, tag_to_elements, volume_to_tags["fluid"], - "wall_interface") - return mesh, tag_to_elements, volume_to_tags - - def my_partitioner(mesh, tag_to_elements, num_ranks): - if use_1d_part: - from mirgecom.simutil import geometric_mesh_partitioner - return geometric_mesh_partitioner( - mesh, num_ranks, auto_balance=True, debug=False) - else: - from meshmode.distributed import get_partition_by_pymetis - return get_partition_by_pymetis(mesh, num_ranks) - - part_func = my_partitioner - - # This bit will write the source decomp's (M) partition table if it - # didn't already exist. We need this table to create the - # N-parted restart data from the M-parted data. - if generate_source_decomp: - if rank == 0: - print("Generating source decomp...") - source_mesh_data = get_mesh_data() - from meshmode.mesh import Mesh - if isinstance(source_mesh_data, Mesh): - multivolume_dataset = False - source_mesh = source_mesh_data - tag_to_elements = None - volume_to_tags = None - elif isinstance(source_mesh_data, tuple): - source_mesh, tag_to_elements, volume_to_tags = source_mesh_data - multivolume_dataset = True - else: - raise TypeError("Unexpected result from get_mesh_data") - comm.bcast(multivolume_dataset) - rank_per_element = my_partitioner(source_mesh, tag_to_elements, mdist) - with open(source_decomp_map_file, "wb") as pkl_file: - pickle.dump(rank_per_element, pkl_file) - print("Done generating source decomp.") - else: - multivolume_dataset = comm.bcast(None) + multivolume_dataset = _is_mesh_data_multivol(src_mesh_filename, mdist) + _validate_mesh_data_files(src_mesh_filename, mdist) + _validate_mesh_data_files(trg_mesh_filename, ndist) - comm.Barrier() + src_mesh_root = src_mesh_filename + "_mesh" + src_decomp_filename = src_mesh_root + f"_decomp_np{mdist}.pkl" + src_idecomp_filename = src_mesh_root + f"_idecomp_np{mdist}.pkl" + src_mvdecomp_filename = src_mesh_root + f"_multivol_idecomp_np{mdist}.pkl" - if rank == 0: - meshtype = " multi-volume " if multivolume_dataset else "" - print(f"Partitioning {meshtype} mesh to {ndist} parts, " - f"writing to {mesh_filename}...") - - # This bit creates the N-parted mesh pkl files and partition table - # Should only do this if the n decomp map is not found in the output path. - distribute_mesh_pkl( - comm, get_mesh_data, filename=mesh_filename, - num_target_ranks=ndist, partition_generator_func=part_func, logmgr=logmgr) + trg_mesh_root = trg_mesh_filename + "_mesh" + trg_decomp_filename = trg_mesh_root + f"_decomp_np{ndist}.pkl" + trg_idecomp_filename = trg_mesh_root + f"_idecomp_np{ndist}.pkl" + trg_mvdecomp_filename = trg_mesh_root + f"_multivol_idecomp_np{ndist}.pkl" - comm.Barrier() + with open(src_idecomp_filename, "rb") as pkl_file: + src_idcmp = pickle.load(pkl_file) + with open(trg_idecomp_filename, "rb") as pkl_file: + trg_idcmp = pickle.load(pkl_file) if rank == 0: - print("Done partitioning target mesh, mesh pkl files written.") - print(f"Generating the restart data for {ndist} parts...") + print("Generating new restart data.") if multivolume_dataset: - target_decomp_map_file = mesh_filename + f"_decomp_np{ndist}.pkl" - target_multivol_decomp_map_file = \ - mesh_filename + f"_multivol_idecomp_np{ndist}.pkl" - with open(target_decomp_map_file, "rb") as pkl_file: - trg_dcmp = pickle.load(pkl_file) - trg_idcmp = invert_decomp(trg_dcmp) - with open(target_multivol_decomp_map_file, "rb") as pkl_file: + with open(trg_mvdecomp_filename, "rb") as pkl_file: trg_mv_dcmp = pickle.load(pkl_file) - source_decomp_map_file = input_path + f"_decomp_np{mdist}.pkl" - source_multivol_decomp_map_file = \ - input_path + f"_multivol_idcomp_np{mdist}.pkl" - with open(source_decomp_map_file, "rb") as pkl_file: - src_dcmp = pickle.load(pkl_file) - src_idcmp = invert_decomp(src_dcmp) - with open(source_multivol_decomp_map_file, "rb") as pkl_file: + with open(src_mvdecomp_filename, "rb") as pkl_file: src_mv_dcmp = pickle.load(pkl_file) from mirgecom.restart import redistribute_multivolume_restart_data redistribute_multivolume_restart_data( actx, comm, src_idcmp, trg_idcmp, src_mv_dcmp, trg_mv_dcmp, input_path, - output_path, mesh_filename) + output_path, trg_mesh_filename) else: - target_decomp_map_file = mesh_filename + f"_decomp_np{ndist}.pkl" from mirgecom.restart import redistribute_restart_data - redistribute_restart_data(actx, comm, source_decomp_map_file, input_path, - target_decomp_map_file, output_path, mesh_filename) + redistribute_restart_data(actx, comm, src_decomp_filename, input_path, + trg_decomp_filename, output_path, trg_mesh_root) if rank == 0: print("Done generating restart data.") @@ -362,28 +291,25 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): level=logging.INFO) parser = argparse.ArgumentParser( - description="MIRGE-Com Mesh Distribution") - parser.add_argument("-w", "--wall", dest="use_wall", - action="store_true", help="Include wall domain in mesh.") - parser.add_argument("-1", "--1dpart", dest="one_d_part", - action="store_true", help="Use 1D partitioner.") + description="MIRGE-Com Restart redistribution utility.") parser.add_argument("-n", "--ndist", type=int, dest="ndist", nargs="?", action="store", help="Number of distributed parts") parser.add_argument("-m", "--mdist", type=int, dest="mdist", nargs="?", action="store", help="Number of source data parts") - parser.add_argument("-s", "--source", type=str, dest="source", - nargs="?", action="store", help="Gmsh mesh source file") parser.add_argument("-o", "--ouput-path", type=str, dest="output_path", nargs="?", action="store", help="Output directory for distributed mesh pkl files") + parser.add_argument("-s", "--source-mesh", type=str, dest="src_mesh", + nargs="?", action="store", + help="Path/filename for source (m parts) mesh.") + parser.add_argument("-t", "--target-mesh", type=str, dest="trg_mesh", + nargs="?", action="store", + help="Path/filename for target (n parts) mesh.") parser.add_argument("-i", "--input-path", type=str, dest="input_path", nargs="?", action="store", help="Input path/root filename for restart pkl files") - parser.add_argument("-c", "--casename", type=str, dest="casename", nargs="?", - action="store", - help="Root name of distributed mesh pkl files.") parser.add_argument("-g", "--logpath", type=str, dest="log_path", nargs="?", action="store", help="simulation case name") @@ -393,8 +319,8 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): actx_class = get_reasonable_array_context_class( lazy=False, distributed=True, profiling=False, numpy=False) - main(actx_class, mesh_source=args.source, - output_path=args.output_path, ndist=args.ndist, - input_path=args.input_path, mdist=args.mdist, - log_path=args.log_path, casename=args.casename, - use_1d_part=args.one_d_part, use_wall=args.use_wall) + main(actx_class, ndist=args.ndist, mdist=args.mdist, + output_path=args.output_path, input_path=args.input_path, + src_mesh_filename=args.src_mesh, + trg_mesh_filename=args.trg_mesh, + log_path=args.log_path) From 89ea9ab353a6453c9f1afad8cb8d56428e11e377 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 11 Oct 2023 12:13:12 -0500 Subject: [PATCH 36/55] Deep refactor to simplify overlap mapping, processing. --- mirgecom/restart.py | 142 ++++++++++++++++++++++++++++---------------- 1 file changed, 91 insertions(+), 51 deletions(-) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index d4576a83a..34825aae3 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -37,6 +37,7 @@ invert_decomp, interdecomposition_overlap, multivolume_interdecomposition_overlap, + # construct_element_mapping, copy_mapped_dof_array_data ) @@ -235,7 +236,9 @@ def _find_rank_with_all_volumes(multivol_decomp_map): ranks_seen_volumes = defaultdict(set) for partid, elements in multivol_decomp_map.items(): - if elements: # non-empty means this rank has data for this volume + if elements is None: + continue + if len(elements) > 0: # non-empty means this rank has data for this volume ranks_seen_volumes[partid.rank].add(partid.volume_tag) # Now, find a rank that has seen all volumes @@ -249,8 +252,8 @@ def _find_rank_with_all_volumes(multivol_decomp_map): # Traverse the restart_data and copy data from the src # into the target DOFArrays in-place, so that the trg data # is persistent across multiple calls with different src data. -def _recursive_map_and_copy(trg_item, src_item, trg_partid_to_index_map, - src_volume_sizes): +def _recursive_map_and_copy(trg_item, src_item, trsrs_idx_maps, + src_volume_sizes, trg_rank, src_rank): """ Recursively map and copy DOFArrays from the source item to the target item. @@ -271,39 +274,48 @@ def _recursive_map_and_copy(trg_item, src_item, trg_partid_to_index_map, The target item after mapping and copying the data from the source item. """ if trg_item is None: - print(f"{src_item=}") - raise ValueError("trg_item is None, but src_item is not.") + # print(f"dbg {src_item=}") + # print("dbg trg_item was None.") + return trg_item + # raise ValueError("trg_item is None, but src_item is not.") if src_item is None: print(f"{trg_item=}") raise ValueError("src_item is None, but trg_item is not.") - trg_rank = next(iter(trg_partid_to_index_map)).rank - + # print(f"{trsrs_idx_maps=}") + # trg_rank = next(iter(trsrs_idx_maps)).rank # print(f"{trg_item=}") # print(f"{src_item=}") - src_nel, src_nnodes = src_item[0].shape - volume_tag = next((vol_tag for vol_tag, size in src_volume_sizes.items() - if size == src_nel), None) - target_partid = PartID(volume_tag=volume_tag, rank=trg_rank) - elem_map = trg_partid_to_index_map[target_partid] - if isinstance(src_item, DOFArray): if trg_item is None: - raise ValueError("No corresponding target DOFArray found.") - return copy_mapped_dof_array_data(trg_item, src_item, elem_map) + raise ValueError( + f"No corresponding target DOFArray found {src_item=}.") + src_nel, src_nnodes = src_item[0].shape + volume_tag = next((vol_tag for vol_tag, size in src_volume_sizes.items() + if size == src_nel), None) + trg_partid = PartID(volume_tag=volume_tag, rank=trg_rank) + if trg_partid in trsrs_idx_maps: + trvs_mapping = trsrs_idx_maps[trg_partid] + src_partid = PartID(volume_tag=volume_tag, rank=src_rank) + elem_map = trvs_mapping[src_partid] + # print(f"dbg Copying data {src_partid=}, {trg_partid=}") + return copy_mapped_dof_array_data(trg_item, src_item, elem_map) + # else: + # print("dbg Skipping copy for non-target volume.") elif isinstance(src_item, dict): return {k: _recursive_map_and_copy( - trg_item.get(k, None), v, trg_partid_to_index_map, - src_volume_sizes) for k, v in src_item.items()} + trg_item.get(k, None), v, trsrs_idx_maps, + src_volume_sizes, trg_rank, src_rank) for k, v in src_item.items()} elif isinstance(src_item, (list, tuple)): return type(src_item)(_recursive_map_and_copy( - t, v, trg_partid_to_index_map, src_volume_sizes) + t, v, trsrs_idx_maps, src_volume_sizes, trg_rank, src_rank) for t, v in zip(trg_item, src_item)) elif is_dataclass(src_item): trg_dict = asdict(trg_item) return type(src_item)(**{ k: _recursive_map_and_copy( - trg_dict.get(k, None), v, trg_partid_to_index_map, - src_volume_sizes) for k, v in asdict(src_item).items()}) + trg_dict.get(k, None), v, trsrs_idx_maps, + src_volume_sizes, trg_rank, src_rank) + for k, v in asdict(src_item).items()}) else: return src_item # dupe non-dof data outright @@ -311,10 +323,10 @@ def _recursive_map_and_copy(trg_item, src_item, trg_partid_to_index_map, def _ensure_unique_nelems(mesh_data_dict): seen_nelems = set() for volid, mesh_data in mesh_data_dict.items(): - if mesh_data.nelements in seen_nelems: + if mesh_data[0].nelements in seen_nelems: raise ValueError(f"Multiple volumes {volid} found with same " "number of elements.") - seen_nelems.add(mesh_data.nelem) + seen_nelems.add(mesh_data[0].nelements) def _get_volume_sizes_on_each_rank(multivol_decomp_map): @@ -383,35 +395,52 @@ def _get_volume_sizes_for_rank(target_rank, multivol_decomp_map): return volume_sizes +def _extract_src_rank_specific_mapping(trs_olaps, src_rank): + """Extract source rank-specific mappings from overlap mapping.""" + return {trg_partid: {src_partid: local_mapping + for src_partid, local_mapping in src_mappings.items() + if src_partid.rank == src_rank} + for trg_partid, src_mappings in trs_olaps.items() + if src_rank in {src_partid.rank for src_partid in src_mappings.keys()}} + + +# Call this one with target-rank-specific mappings (trs_olaps) of the form: +# {targ_partid : { src_partid : {trg_el_index : src_el_index} } } def _get_restart_data_for_target_rank( - actx, trg_rank, sample_rst_data, sample_vol_sizes, src_overlaps, + actx, trg_rank, sample_rst_data, sample_vol_sizes, trs_olaps, target_multivol_decomp_map, source_multivol_decomp_map, input_path): trg_vol_sizes = _get_volume_sizes_for_rank( trg_rank, target_multivol_decomp_map) - + # print(f"dbg Target rank = {trg_rank}") with array_context_for_pickling(actx): out_rst_data = _recursive_resize_reinit_with_zeros( actx, sample_rst_data, trg_vol_sizes, sample_vol_sizes) + src_ranks_in_mapping = {k.rank for v in trs_olaps.values() + for k in v.keys()} + # print(f"dbg olap src ranks: {src_ranks_in_mapping}") + + # Read and Map DOFArrays from each source rank to target + for src_rank in src_ranks_in_mapping: + # get src_rank-specific overlaps + trsrs_idx_maps = _extract_src_rank_specific_mapping(trs_olaps, src_rank) + # print(f"dbg {src_rank=},{trsrs_idx_maps=}") + src_vol_sizes = _get_volume_sizes_for_rank( + src_rank, source_multivol_decomp_map) + src_restart_file = f"{input_path}-{src_rank:04d}.pkl" + with array_context_for_pickling(actx): + with open(src_restart_file, "rb") as f: + src_rst_data = pickle.load(f) + src_mesh_data = src_rst_data.pop("volume_to_local_mesh_data", None) + _ensure_unique_nelems(src_mesh_data) + # Copies data for all overlapping parts from src to trg rank data + with array_context_for_pickling(actx): + out_rst_data = \ + _recursive_map_and_copy( + out_rst_data, src_rst_data, trsrs_idx_maps, + src_vol_sizes, trg_rank, src_rank) - # Read and Map DOFArrays from source to target - for src_rank, trg_partid_to_idx_map in src_overlaps.items(): - src_vol_sizes = _get_volume_sizes_for_rank( - src_rank, source_multivol_decomp_map) - - src_restart_file = f"{input_path}-{src_rank:04d}.pkl" - with array_context_for_pickling(actx): - with open(src_restart_file, "rb") as f: - src_rst_data = pickle.load(f) - mesh_data = src_rst_data.pop("volume_to_local_mesh_data", None) - _ensure_unique_nelems(mesh_data) - - with array_context_for_pickling(actx): - out_rst_data = _recursive_map_and_copy( - out_rst_data, src_rst_data, trg_partid_to_idx_map, - src_vol_sizes) - - return out_rst_data + return out_rst_data def redistribute_multivolume_restart_data( @@ -462,7 +491,7 @@ def redistribute_multivolume_restart_data( sample_rank = _find_rank_with_all_volumes(source_multivol_decomp_map) if sample_rank is None: raise ValueError("No source rank found with data for all volumes.") - + print(f"Found source rank {sample_rank} having data for all volumes.") mesh_data_item = "volume_to_local_mesh_data" sample_restart_file = f"{src_input_path}-{sample_rank:04d}.pkl" with array_context_for_pickling(actx): @@ -486,7 +515,7 @@ def redistribute_multivolume_restart_data( if writer_color: writer_nprocs = writer_comm_wrapper.Get_size() writer_rank = writer_comm_wrapper.Get_rank() - nparts_per_writer = int(writer_nprocs / trg_nparts) + nparts_per_writer = max(1, trg_nparts // writer_nprocs) nleftover = trg_nparts - (nparts_per_writer * writer_nprocs) nparts_this_writer = nparts_per_writer + (1 if writer_rank < nleftover else 0) @@ -495,21 +524,32 @@ def redistribute_multivolume_restart_data( < nleftover else nleftover) my_ending_rank = my_starting_rank + nparts_this_writer - 1 parts_to_write = list(range(my_starting_rank, my_ending_rank+1)) - xdo = multivolume_interdecomposition_overlap(target_idecomp_map, - source_idecomp_map, - target_multivol_decomp_map, + print(f"{my_rank=}, {writer_rank=}, {parts_to_write=}") + # print(f"{source_multivol_decomp_map=}") + # print(f"{target_multivol_decomp_map=}") + + xdo = multivolume_interdecomposition_overlap(source_idecomp_map, + target_idecomp_map, source_multivol_decomp_map, + target_multivol_decomp_map, return_ranks=parts_to_write) - for trg_rank, olaps in xdo.items(): + # for trg_partid, olaps in xdo.items(): + # print(f"dbg {trg_partid=}, " + # "f{len(target_multivol_map[trg_partid])=}") + # for src_partid, elem_map in olaps.items(): + # print(f"dbg {src_partid=}, {len(elem_map)=}") + for trg_rank in parts_to_write: + trs_olaps = {k: v for k, v in xdo.items() if k.rank == trg_rank} out_rst_data = _get_restart_data_for_target_rank( - actx, trg_rank, sample_rst_data, sample_vol_sizes, - olaps, target_multivol_decomp_map, source_multivol_decomp_map, + actx, trg_rank, sample_rst_data, sample_vol_sizes, trs_olaps, + target_multivol_decomp_map, source_multivol_decomp_map, src_input_path) # Read new mesh data and stack it in the restart file - mesh_pkl_filename = f"{mesh_filename}_np{trg_nparts}_rank{trg_rank}.pkl" + mesh_pkl_filename = \ + f"{mesh_filename}_mesh_np{trg_nparts}_rank{trg_rank}.pkl" with array_context_for_pickling(actx): with open(mesh_pkl_filename, "rb") as pkl_file: global_nelements, mesh_data = \ From 43b29901776ce3a591609f075e18d4f982dbd999 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 11 Oct 2023 12:13:26 -0500 Subject: [PATCH 37/55] Deep refactor to simplify overlap mapping, processing. --- mirgecom/simutil.py | 137 ++++++++++++++++++++++++++++---------------- 1 file changed, 88 insertions(+), 49 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 4669b0b57..ba131b968 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -1345,14 +1345,15 @@ def partition_generator_func(mesh, tag_to_elements, num_target_ranks): part_id_to_elements = _get_multi_volume_partitions( mesh, num_target_ranks, rank_per_element, tag_to_elements, volume_to_tags) + # Save this little puppy for later (m-to-n restart support) if reader_rank == 0: mv_part_table_fname = \ filename + f"_multivol_idecomp_np{num_target_ranks}.pkl" if os.path.exists(mv_part_table_fname): os.remove(mv_part_table_fname) - with open(mv_part_table_fname, "wb") as pkl_file: - pickle.dump(part_id_to_elements, pkl_file) + with open(mv_part_table_fname, "wb") as pkl_file: + pickle.dump(part_id_to_elements, pkl_file) reader_comm.Barrier() if reader_rank == 0: @@ -1608,71 +1609,109 @@ def interdecomposition_overlap(target_decomp_map, source_decomp_map, # Interdecomposition overlap utility for multi-volume datasets -def multivolume_interdecomposition_overlap(target_idecomp, source_idecomp, - target_vol_decomp, source_vol_decomp, - return_ranks=None): +def multivolume_interdecomposition_overlap(src_decomp_map, trg_decomp_map, + src_multivol_decomp_map, trg_multivol_decomp_map, + return_ranks=None): """ - Map element indices for overlapping, disparate decompositions with volumes. + Construct local-to-local index mapping for overlapping deconmps. Parameters ---------- - target_idecomp: Decomposition map of the target {rank: [elements]} - source_idecomp: Decomposition map of the source {rank: [elements]} - target_vol_decomp: Decomposition map of the target with volumes - {PartID: [elements]} - source_vol_decomp: Decomposition map of the source with volumes - {PartID: [elements]} - return_ranks: List of ranks for which the overlaps should be computed. - If None, all ranks are considered. + src_decomp_map: dict + Source decomposition map {rank: [elements]} + trg_decomp_map: dict + Target decomposition map {rank: [elements]} + src_multivol_decomp_map: dict + Source multivolume decomposition map {PartID: np.array(elements)} + trg_multivol_decomp_map: dict + Target multivolume decomposition map {PartID: np.array(elements)} Returns ------- A dictionary with structure: { - target_rank: { - source_rank: { - (targ)PartID: { - target_local_index: source_local_index - } + trg_partid: { + src_partid: { + trg_local_el_index: src_local_el_index } } } """ # If no specific ranks are provided, consider all ranks in the target decomp if return_ranks is None: - return_ranks = list(target_idecomp.keys()) - - rank_overlap_map = interdecomposition_imapping(target_idecomp, source_idecomp) - - overlap_maps = {} - - for trg_rank in return_ranks: - # Extract set of unique PartIDs for current trg rank from tgt_vol_decomp - targ_partids = [partid for partid in target_vol_decomp.keys() - if partid.rank == trg_rank] - - overlap_maps[trg_rank] = {} - for src_rank in rank_overlap_map[trg_rank]: - overlap_maps[trg_rank][src_rank] = {} + return_ranks = list(trg_decomp_map.keys()) - for targ_partid in targ_partids: - src_partid = PartID(volume_tag=targ_partid.volume_tag, rank=src_rank) - overlap_maps[trg_rank][src_rank][targ_partid] = {} + mapping = {} - # Determine element overlaps, set is used for performance considerations - target_elements_set = set(target_vol_decomp[targ_partid]) - source_elements_set = set(source_vol_decomp.get(src_partid, [])) + # First, identify overlapping ranks using the regular decomp maps + overlapping_ranks = interdecomposition_imapping(trg_decomp_map, src_decomp_map) + # print(f"{overlapping_ranks=}") - common_elements_set = \ - target_elements_set.intersection(source_elements_set) - - for trg_el in common_elements_set: - trg_local_idx = target_vol_decomp[targ_partid].index(trg_el) - src_local_idx = source_vol_decomp[src_partid].index(trg_el) - overlap_maps[trg_rank][src_rank][targ_partid][trg_local_idx] = \ - src_local_idx - - return overlap_maps + # Now, for each overlapping rank, determine the overlapping elements using the + # multivol decomp maps + for trg_partid, trg_elems in trg_multivol_decomp_map.items(): + trg_nelem_part = len(trg_elems) + # print(f"dbg Finding overlaps for: {trg_partid=}, {trg_nelem_part}") + if trg_nelem_part == 0: + # print("dbg - Skipping empty target part.") + continue + trg_rank = trg_partid.rank + if trg_rank not in return_ranks: + # print("dbg - Skipping unrelated trg rank.") + continue + target_elements_set = set(trg_elems) + # print(f"dbg {target_elements_set=}") + # print(f"dbg type of trg_elems={type(trg_elems)}, " + # f"trg_elems content={trg_elems}") + noverlap = 0 + for src_rank in overlapping_ranks[trg_rank]: + # print(f"dbg - Searching for src partids with {src_rank=}") + for src_partid, src_elems in src_multivol_decomp_map.items(): + # print(f"dbg -- Considering {src_partid=}, {len(src_elems)=}") + if src_partid.rank != src_rank: + # print("dbg --- Skipping unrelated src rank.") + continue + if src_partid.volume_tag != trg_partid.volume_tag: + # print("dbg --- Skipping unrelated src volume.") + continue + # print("dbg --- Determining overlap") + # Determine element overlaps, set is used for performance + source_elements_set = set(src_elems) + # print(f"dbg {source_elements_set=}") + common_elements_set = \ + target_elements_set.intersection(source_elements_set) + # print(f"dbg {common_elements_set=}") + if common_elements_set: + if trg_partid not in mapping: + mapping[trg_partid] = {} + + local_mapping = {} + for trg_el in common_elements_set: + # print(f"dbg {trg_el=}") + trg_local_idx = np.where(trg_elems == trg_el)[0][0] + src_local_idx = np.where(src_elems == trg_el)[0][0] + local_mapping[trg_local_idx] = src_local_idx + + # Store the local mapping if there are any overlapping elements + if local_mapping: + # init empty dict if needed + # if trg_partid not in mapping: + # mapping[trg_partid] = {} + num_local_overlap = len(local_mapping) + noverlap = noverlap + num_local_overlap + # print(f"dbg ---- found overlap: {trg_partid=}, {src_partid=}") + # print(f"dbg ---- num_olap: {num_local_overlap}") + mapping[trg_partid][src_partid] = local_mapping + # else: + # print("dbg ---- No overlap found.") + # if noverlap == trg_nelem_part: + # print("dbg - Full overlaps found.") + if noverlap != trg_nelem_part: + # print("dbg - Overlaps did not cover target part!") + raise AssertionError("Source overlaps did not cover target part." + f" {trg_partid=}") + + return mapping def boundary_report(dcoll, boundaries, outfile_name, *, dd=DD_VOLUME_ALL, From 0ceb0818c5306af2c3d5c2b4291042e07896111a Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 11 Oct 2023 12:14:03 -0500 Subject: [PATCH 38/55] More extensive testing to catch errors encountered in prediction case. --- test/test_restart.py | 144 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 114 insertions(+), 30 deletions(-) diff --git a/test/test_restart.py b/test/test_restart.py index f97bf450e..e71954e4a 100644 --- a/test/test_restart.py +++ b/test/test_restart.py @@ -28,6 +28,7 @@ import numpy.random import logging import pytest +from collections import defaultdict from pytools.obj_array import make_obj_array from mirgecom.discretization import create_discretization_collection from meshmode.array_context import ( # noqa @@ -214,6 +215,7 @@ def test_dofarray_mapped_copy(actx_factory): def test_multivolume_interdecomp_overlap_basic(): """Test the multivolume_interdecomp_overlap.""" + import numpy as np # Total elements in the testing mesh total_elements = 100 @@ -237,8 +239,8 @@ def test_multivolume_interdecomp_overlap_basic(): # Testing volume decomps # Vol1: even elements, Vol2: odd elements - vol1_elements = [i for i in range(total_elements) if i % 2 == 0] - vol2_elements = [i for i in range(total_elements) if i % 2 != 0] + vol1_elements = np.array([i for i in range(total_elements) if i % 2 == 0]) + vol2_elements = np.array([i for i in range(total_elements) if i % 2 != 0]) src_vol_decomp = {} trg_vol_decomp = {} @@ -246,34 +248,73 @@ def test_multivolume_interdecomp_overlap_basic(): # Test vol decomps for i in range(src_np): src_vol_decomp[PartID(volume_tag="vol1", rank=i)] = \ - [el for el in src_decomp[i] if el in vol1_elements] + np.array([el for el in src_decomp[i] if el in vol1_elements]) src_vol_decomp[PartID(volume_tag="vol2", rank=i)] = \ - [el for el in src_decomp[i] if el in vol2_elements] + np.array([el for el in src_decomp[i] if el in vol2_elements]) for i in range(trg_np): trg_vol_decomp[PartID(volume_tag="vol1", rank=i)] = \ - [el for el in trg_decomp[i] if el in vol1_elements] + np.array([el for el in trg_decomp[i] if el in vol1_elements]) trg_vol_decomp[PartID(volume_tag="vol2", rank=i)] = \ - [el for el in trg_decomp[i] if el in vol2_elements] + np.array([el for el in trg_decomp[i] if el in vol2_elements]) from mirgecom.simutil import multivolume_interdecomposition_overlap # Compute the multivolume interdecomp overlaps mv_idx = multivolume_interdecomposition_overlap( - trg_decomp, src_decomp, trg_vol_decomp, src_vol_decomp + src_decomp, trg_decomp, src_vol_decomp, trg_vol_decomp ) - # Perform some basic checks on the returned data structure - # The num trg ranks in the overlap should match the input - assert set(mv_idx.keys()) == set(trg_decomp.keys()) - - for _trg_rank, src_rank_mappings in mv_idx.items(): - for src_rank, partid_mappings in src_rank_mappings.items(): - for partid, element_mappings in partid_mappings.items(): - for trg_local_idx, src_local_idx in element_mappings.items(): - # match element ids for the trg and src at resp index - assert trg_vol_decomp[partid][trg_local_idx] == \ - src_vol_decomp[PartID(volume_tag=partid.volume_tag, - rank=src_rank)][src_local_idx] + for trg_partid, src_partid_mappings in mv_idx.items(): + uncovered_elements = set(trg_vol_decomp[trg_partid]) + for src_partid, element_mapping in src_partid_mappings.items(): + for trg_local_idx, _ in element_mapping.items(): + uncovered_elements.discard(trg_vol_decomp[trg_partid][trg_local_idx]) + for trg_local_idx, src_local_idx in element_mapping.items(): + # match element ids for the trg and src at resp index + assert trg_vol_decomp[trg_partid][trg_local_idx] == \ + src_vol_decomp[src_partid][src_local_idx] + assert not uncovered_elements + + for trg_partid, src_partid_mappings in mv_idx.items(): + mapped_trg_elems = set() + # validate ranks in trg mapping + assert 0 <= trg_partid.rank < trg_np,\ + f"Invalid target rank: {trg_partid.rank}" + for src_partid, element_mapping in src_partid_mappings.items(): + # check for consistent volume_tags + assert trg_partid.volume_tag == src_partid.volume_tag, \ + f"Volume tag mismatch: {trg_partid.volume_tag} "\ + f"vs {src_partid.volume_tag}" + # validate ranks in src mapping + assert 0 <= src_partid.rank < src_np,\ + f"Invalid source rank: {src_partid.rank}" + for trg_local_idx, src_local_idx in element_mapping.items(): + # check that each trg el is mapped only once + assert trg_local_idx not in mapped_trg_elems,\ + f"Duplicate mapping for target element {trg_local_idx}" + mapped_trg_elems.add(trg_local_idx) + # check for valid src and trg indices + assert 0 <= src_local_idx < len(src_vol_decomp[src_partid]),\ + f"Invalid source index {src_local_idx} for {src_partid}" + assert 0 <= trg_local_idx < len(trg_vol_decomp[trg_partid]),\ + f"Invalid target index {trg_local_idx} for {trg_partid}" + + # Check that the mapping is 1-to-1, that each src element is covered and maps + # to one and only one target element + accumulated_mapped_src_elems = defaultdict(list) + for _, src_partid_mappings in mv_idx.items(): + for src_partid, element_mapping in src_partid_mappings.items(): + mapped_src_elems = list(element_mapping.values()) + accumulated_mapped_src_elems[src_partid].extend(mapped_src_elems) + + for src_partid, mapped_src_elems in accumulated_mapped_src_elems.items(): + src_elem_set = set(mapped_src_elems) + assert set(range(len(src_vol_decomp[src_partid]))) == src_elem_set, \ + f"Some elements in {src_partid} are not mapped to any target element" + # do not map to more than one trg elem + for elem in src_elem_set: + assert mapped_src_elems.count(elem) == 1, \ + f"Element {elem} in {src_partid} is mapped more than once" def _generate_decompositions(total_elements, num_ranks, pattern="chunked"): @@ -295,7 +336,8 @@ def _generate_decompositions(total_elements, num_ranks, pattern="chunked"): decomp = {i: all_elements[i * elements_per_rank: (i + 1) * elements_per_rank] for i in range(num_ranks)} decomp[num_ranks - 1].extend(all_elements[num_ranks * elements_per_rank:]) - return decomp + + return {k: np.array(v) for k, v in decomp.items()} # Convert to numpy arrays @pytest.mark.parametrize("decomp_pattern", ["chunked", "strided", "random"]) @@ -343,15 +385,57 @@ def test_multivolume_interdecomp_overlap(decomp_pattern, vol_pattern, src_trg_ra # Testing the overlap utility from mirgecom.simutil import multivolume_interdecomposition_overlap mv_idx = multivolume_interdecomposition_overlap( - trg_decomp, src_decomp, trg_vol_decomp, src_vol_decomp + src_decomp, trg_decomp, src_vol_decomp, trg_vol_decomp ) - assert set(mv_idx.keys()) == set(trg_decomp.keys()) - - for _trg_rank, src_rank_mappings in mv_idx.items(): - for src_rank, partid_mappings in src_rank_mappings.items(): - for partid, element_mappings in partid_mappings.items(): - for trg_local_idx, src_local_idx in element_mappings.items(): - assert trg_vol_decomp[partid][trg_local_idx] == \ - src_vol_decomp[PartID(volume_tag=partid.volume_tag, - rank=src_rank)][src_local_idx] + for trg_partid, src_partid_mappings in mv_idx.items(): + uncovered_elements = set(trg_vol_decomp[trg_partid]) + for src_partid, element_mapping in src_partid_mappings.items(): + for trg_local_idx, _ in element_mapping.items(): + uncovered_elements.discard(trg_vol_decomp[trg_partid][trg_local_idx]) + for trg_local_idx, src_local_idx in element_mapping.items(): + # match element ids for the trg and src at resp index + assert trg_vol_decomp[trg_partid][trg_local_idx] == \ + src_vol_decomp[src_partid][src_local_idx] + assert not uncovered_elements + + for trg_partid, src_partid_mappings in mv_idx.items(): + mapped_trg_elems = set() + # validate ranks in trg mapping + assert 0 <= trg_partid.rank < trg_np,\ + f"Invalid target rank: {trg_partid.rank}" + for src_partid, element_mapping in src_partid_mappings.items(): + # check for consistent volume_tags + assert trg_partid.volume_tag == src_partid.volume_tag, \ + f"Volume tag mismatch: {trg_partid.volume_tag} "\ + f"vs {src_partid.volume_tag}" + # validate ranks in src mapping + assert 0 <= src_partid.rank < src_np,\ + f"Invalid source rank: {src_partid.rank}" + for trg_local_idx, src_local_idx in element_mapping.items(): + # check that each trg el is mapped only once + assert trg_local_idx not in mapped_trg_elems,\ + f"Duplicate mapping for target element {trg_local_idx}" + mapped_trg_elems.add(trg_local_idx) + # check for valid src and trg indices + assert 0 <= src_local_idx < len(src_vol_decomp[src_partid]),\ + f"Invalid source index {src_local_idx} for {src_partid}" + assert 0 <= trg_local_idx < len(trg_vol_decomp[trg_partid]),\ + f"Invalid target index {trg_local_idx} for {trg_partid}" + + # Check that the mapping is 1-to-1, that each src element is covered and maps + # to one and only one target element + accumulated_mapped_src_elems = defaultdict(list) + for _, src_partid_mappings in mv_idx.items(): + for src_partid, element_mapping in src_partid_mappings.items(): + mapped_src_elems = list(element_mapping.values()) + accumulated_mapped_src_elems[src_partid].extend(mapped_src_elems) + + for src_partid, mapped_src_elems in accumulated_mapped_src_elems.items(): + src_elem_set = set(mapped_src_elems) + assert set(range(len(src_vol_decomp[src_partid]))) == src_elem_set, \ + f"Some elements in {src_partid} are not mapped to any target element" + # do not map to more than one trg elem + for elem in src_elem_set: + assert mapped_src_elems.count(elem) == 1, \ + f"Element {elem} in {src_partid} is mapped more than once" From 98c114f23b8710ad6175491b8fcefa98f5cc3a54 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 11 Oct 2023 18:10:45 -0500 Subject: [PATCH 39/55] Add a bunch of debugging/diagnostics for restart data structures and decomps --- mirgecom/restart.py | 140 ++++++++++++++++++++++++++++++++++++++------ mirgecom/simutil.py | 42 +++++++++++++ 2 files changed, 163 insertions(+), 19 deletions(-) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index 34825aae3..872152c3e 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -29,6 +29,7 @@ """ import pickle +import numpy as np from meshmode.dof_array import array_context_for_pickling, DOFArray from grudge.discretization import PartID from dataclasses import is_dataclass, asdict @@ -37,9 +38,10 @@ invert_decomp, interdecomposition_overlap, multivolume_interdecomposition_overlap, - # construct_element_mapping, + summarize_decomposition, copy_mapped_dof_array_data ) +from pprint import pprint class PathError(RuntimeError): @@ -253,7 +255,8 @@ def _find_rank_with_all_volumes(multivol_decomp_map): # into the target DOFArrays in-place, so that the trg data # is persistent across multiple calls with different src data. def _recursive_map_and_copy(trg_item, src_item, trsrs_idx_maps, - src_volume_sizes, trg_rank, src_rank): + src_volume_sizes, trg_rank, src_rank, + data_path=None): """ Recursively map and copy DOFArrays from the source item to the target item. @@ -273,18 +276,24 @@ def _recursive_map_and_copy(trg_item, src_item, trsrs_idx_maps, object: The target item after mapping and copying the data from the source item. """ + if data_path is None: + data_path = [] + path_string = f"{' -> '.join(map(str, data_path))}" if trg_item is None: - # print(f"dbg {src_item=}") - # print("dbg trg_item was None.") - return trg_item - # raise ValueError("trg_item is None, but src_item is not.") + print(f"Target {trg_rank=} item is None at path: {path_string}.") + print(f"Source {src_rank=} item at this path is: {type(src_item)}") + # return trg_item + raise ValueError("trg_item is None, but src_item is not.") + if src_item is None: print(f"{trg_item=}") raise ValueError("src_item is None, but trg_item is not.") + # print(f"{trsrs_idx_maps=}") # trg_rank = next(iter(trsrs_idx_maps)).rank # print(f"{trg_item=}") # print(f"{src_item=}") + if isinstance(src_item, DOFArray): if trg_item is None: raise ValueError( @@ -292,6 +301,8 @@ def _recursive_map_and_copy(trg_item, src_item, trsrs_idx_maps, src_nel, src_nnodes = src_item[0].shape volume_tag = next((vol_tag for vol_tag, size in src_volume_sizes.items() if size == src_nel), None) + if not volume_tag: + raise ValueError(f"Could not resolve src volume for {path_string}.") trg_partid = PartID(volume_tag=volume_tag, rank=trg_rank) if trg_partid in trsrs_idx_maps: trvs_mapping = trsrs_idx_maps[trg_partid] @@ -299,22 +310,38 @@ def _recursive_map_and_copy(trg_item, src_item, trsrs_idx_maps, elem_map = trvs_mapping[src_partid] # print(f"dbg Copying data {src_partid=}, {trg_partid=}") return copy_mapped_dof_array_data(trg_item, src_item, elem_map) - # else: - # print("dbg Skipping copy for non-target volume.") + else: + # print("dbg Skipping copy for non-target volume.") + return trg_item + elif isinstance(src_item, np.ndarray): + if trg_item is None: + raise ValueError( + f"No corresponding target ndarray found for {src_item=}.") + # Create a new ndarray with the same shape as the src_item + result_array = np.empty_like(src_item, dtype=object) + for idx, array_element in np.ndenumerate(src_item): + result_array[idx] = _recursive_map_and_copy( + trg_item[idx], array_element, trsrs_idx_maps, + src_volume_sizes, trg_rank, src_rank, data_path + [str(idx)]) + + return result_array elif isinstance(src_item, dict): return {k: _recursive_map_and_copy( trg_item.get(k, None), v, trsrs_idx_maps, - src_volume_sizes, trg_rank, src_rank) for k, v in src_item.items()} + src_volume_sizes, trg_rank, src_rank, + data_path + [k])for k, v in src_item.items()} elif isinstance(src_item, (list, tuple)): return type(src_item)(_recursive_map_and_copy( - t, v, trsrs_idx_maps, src_volume_sizes, trg_rank, src_rank) - for t, v in zip(trg_item, src_item)) + t, v, trsrs_idx_maps, src_volume_sizes, trg_rank, + src_rank, + data_path + [str(idx)]) for idx, (t, v) in enumerate(zip(trg_item, + src_item))) elif is_dataclass(src_item): trg_dict = asdict(trg_item) return type(src_item)(**{ k: _recursive_map_and_copy( trg_dict.get(k, None), v, trsrs_idx_maps, - src_volume_sizes, trg_rank, src_rank) + src_volume_sizes, trg_rank, src_rank, data_path + [k]) for k, v in asdict(src_item).items()}) else: return src_item # dupe non-dof data outright @@ -404,6 +431,32 @@ def _extract_src_rank_specific_mapping(trs_olaps, src_rank): if src_rank in {src_partid.rank for src_partid in src_mappings.keys()}} +def _get_item_structure(item): + """Return a simplified representation of the data structure with field names.""" + if isinstance(item, DOFArray): + return "DOFArray" + elif isinstance(item, np.ndarray): + # Create a new ndarray with the same shape as the item + item_structure = np.empty_like(item, dtype=object) + + for idx, array_element in np.ndenumerate(item): + item_structure[idx] = _get_item_structure(array_element) + + return f"ndarray({', '.join(map(str, item_structure.flatten()))})" + elif isinstance(item, dict): + return {k: _get_item_structure(v) for k, v in item.items()} + elif isinstance(item, (list, tuple)): + item_structure = [_get_item_structure(v) for v in item] + return f"{type(item).__name__}({', '.join(item_structure)})" + elif is_dataclass(item): + item_fields = asdict(item) + field_structure = [f"{k}: {_get_item_structure(v)}" + for k, v in item_fields.items()] + return f"{type(item).__name__}({', '.join(field_structure)})" + else: + return type(item).__name__ + + # Call this one with target-rank-specific mappings (trs_olaps) of the form: # {targ_partid : { src_partid : {trg_el_index : src_el_index} } } def _get_restart_data_for_target_rank( @@ -413,14 +466,24 @@ def _get_restart_data_for_target_rank( trg_vol_sizes = _get_volume_sizes_for_rank( trg_rank, target_multivol_decomp_map) # print(f"dbg Target rank = {trg_rank}") - with array_context_for_pickling(actx): - out_rst_data = _recursive_resize_reinit_with_zeros( - actx, sample_rst_data, trg_vol_sizes, sample_vol_sizes) src_ranks_in_mapping = {k.rank for v in trs_olaps.values() for k in v.keys()} # print(f"dbg olap src ranks: {src_ranks_in_mapping}") - # Read and Map DOFArrays from each source rank to target + with array_context_for_pickling(actx): + inp_rst_structure = _get_item_structure(sample_rst_data) + out_rst_data = _recursive_resize_reinit_with_zeros( + actx, sample_rst_data, trg_vol_sizes, sample_vol_sizes) + trg_rst_structure = _get_item_structure(out_rst_data) + # print(f"Initial output restart data structure {trg_rank=}:") + # pprint(trg_rst_structure) + if inp_rst_structure != trg_rst_structure: + print("Initial structure for input:") + pprint(inp_rst_structure) + print(f"Initial structure for {trg_rank=}:") + pprint(trg_rst_structure) + raise AssertionError("Input and output data structure mismatch.") + for src_rank in src_ranks_in_mapping: # get src_rank-specific overlaps trsrs_idx_maps = _extract_src_rank_specific_mapping(trs_olaps, src_rank) @@ -433,12 +496,21 @@ def _get_restart_data_for_target_rank( src_rst_data = pickle.load(f) src_mesh_data = src_rst_data.pop("volume_to_local_mesh_data", None) _ensure_unique_nelems(src_mesh_data) + + # with array_context_for_pickling(actx): + # src_rst_structure = _get_item_structure(src_rst_data) + # print(f"Copying {src_rank=} data to {trg_rank=} with src structure:") + # pprint(src_rst_structure) + # Copies data for all overlapping parts from src to trg rank data with array_context_for_pickling(actx): out_rst_data = \ _recursive_map_and_copy( out_rst_data, src_rst_data, trsrs_idx_maps, src_vol_sizes, trg_rank, src_rank) + # out_rst_structure = _get_item_structure(out_rst_data) + # print(f"After copy of {src_rank=}, {trg_rank=} data structure is:") + # pprint(out_rst_structure) return out_rst_data @@ -458,7 +530,7 @@ def redistribute_multivolume_restart_data( actx: :class:`arraycontext.ArrayContext` The array context used for operations comm: - Am MPI communicator object + An MPI communicator object source_idecomp_map: dict Decomposition map of the source distribution without volume tags. target_idecomp_map: dict @@ -487,21 +559,36 @@ def redistribute_multivolume_restart_data( comm_wrapper = pkl5.Intracomm(comm) my_rank = comm_wrapper.Get_rank() + if my_rank == 0: + print("Redistributing restart data.") + # Give some information about the current partitioning: + print("Source decomp summary:") + summarize_decomposition(source_idecomp_map, source_multivol_decomp_map) + print("\nTarget decomp summary:") + summarize_decomposition(target_idecomp_map, target_multivol_decomp_map) + # Identify a source rank with data for all volumes sample_rank = _find_rank_with_all_volumes(source_multivol_decomp_map) if sample_rank is None: raise ValueError("No source rank found with data for all volumes.") - print(f"Found source rank {sample_rank} having data for all volumes.") - mesh_data_item = "volume_to_local_mesh_data" + if my_rank == 0: + print(f"Found source rank {sample_rank} having data for all volumes.") sample_restart_file = f"{src_input_path}-{sample_rank:04d}.pkl" with array_context_for_pickling(actx): with open(sample_restart_file, "rb") as f: sample_rst_data = pickle.load(f) + mesh_data_item = "volume_to_local_mesh_data" if "mesh" in sample_rst_data: mesh_data_item = "mesh" # check mesh data return type instead vol_to_sample_mesh_data = \ sample_rst_data.pop(mesh_data_item, None) _ensure_unique_nelems(vol_to_sample_mesh_data) + if my_rank == 0: + with array_context_for_pickling(actx): + print("Restart data structure:") + inp_rst_structure = _get_item_structure(sample_rst_data) + pprint(inp_rst_structure) + # sample_vol_sizes, determine from mesh? sample_vol_sizes = _get_volume_sizes_for_rank(sample_rank, source_multivol_decomp_map) @@ -547,6 +634,21 @@ def redistribute_multivolume_restart_data( target_multivol_decomp_map, source_multivol_decomp_map, src_input_path) + with array_context_for_pickling(actx): + if "nparts" in out_rst_data: # reset nparts! + out_rst_data["nparts"] = trg_nparts + if "num_parts" in out_rst_data: # reset nparts! + out_rst_data["num_parts"] = trg_nparts + # print("Output restart structure (sans mesh):") + out_rst_structure = _get_item_structure(out_rst_data) + # pprint(out_rst_structure) + if inp_rst_structure != out_rst_structure: + print("Initial structure for input:") + pprint(inp_rst_structure) + print(f"Output structure for {trg_rank=}:") + pprint(out_rst_structure) + raise AssertionError("Input and output data structure mismatch.") + # Read new mesh data and stack it in the restart file mesh_pkl_filename = \ f"{mesh_filename}_mesh_np{trg_nparts}_rank{trg_rank}.pkl" diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index ba131b968..79c80b70f 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -92,6 +92,7 @@ from grudge.discretization import DiscretizationCollection, PartID from grudge.dof_desc import DD_VOLUME_ALL from meshmode.dof_array import DOFArray +from collections import defaultdict from mirgecom.utils import normalize_boundaries from mirgecom.viscous import get_viscous_timestep @@ -1489,6 +1490,9 @@ def copy_mapped_dof_array_data(trg_dof_array, src_dof_array, index_map): src_nel, src_nnodes = src_array.shape trg_nel, trg_nnodes = trg_array.shape + if trg_nel == 0 or src_nel == 0: + return trg_dof_array + if src_nnodes != trg_nnodes: raise ValueError("DOFArray mapped copy must be of same order.") @@ -1555,6 +1559,44 @@ def interdecomposition_mapping(target_decomp, source_decomp): return interdecomp_map +def summarize_decomposition(decomp_map, multivol_decomp_map): + """Summarize decomp.""" + # Inputs are the decomp_map {rank: [elements]} + # and multivol_decomp_map {PartID: array([elements])} + nranks = len(decomp_map) + + # Initialize counters and containers + total_num_elem = 0 + volume_element_counts = defaultdict(int) + rank_element_counts = {} + rank_volume_element_counts = defaultdict(lambda: defaultdict(int)) + unique_volumes = set() + + # Process data from decomp_map + for rank, elements in decomp_map.items(): + rank_element_counts[rank] = len(elements) + total_num_elem += len(elements) + + # Process data from multivol_decomp_map + for partid, elements in multivol_decomp_map.items(): + vol, rank = partid.volume_tag, partid.rank + unique_volumes.add(vol) + nvol_els = len(elements) + volume_element_counts[vol] += nvol_els + rank_volume_element_counts[rank][vol] = nvol_els + + # Print summary + print(f"Number of elements: {total_num_elem}") + print(f"Volumes({len(unique_volumes)}): {unique_volumes}") + for vol, count in volume_element_counts.items(): + print(f" - Volume({vol}): {count} elements.") + print(f"Number of ranks: {nranks}") + for rank in range(nranks): + print(f" - Rank({rank}): {rank_element_counts[rank]} elements.") + for vol, size in rank_volume_element_counts[rank].items(): + print(f" -- Vol({vol}): {size}") + + # Need a function to determine which of my local elements overlap # with a disparate decomp part. Optionally restrict attention to # selected parts. From 74c0edb93f9f02b581df30af55f592b87b860f38 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 11 Oct 2023 21:02:52 -0500 Subject: [PATCH 40/55] Add more diagnostics, handle ndarrays in zero util --- mirgecom/restart.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index 872152c3e..4d47ec49b 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -396,6 +396,17 @@ def _recursive_resize_reinit_with_zeros(actx, sample_item, target_volume_sizes, zeros_array = actx.zeros(shape=(trg_nel, sample_nnodes), dtype=sample_item[0].dtype) return DOFArray(actx, (zeros_array,)) + elif isinstance(sample_item, np.ndarray): + if sample_item.size > 0 and isinstance(sample_item.flat[0], DOFArray): + # handle ndarray containing DOFArrays + new_shape = sample_item.shape + new_arr = np.empty(new_shape, dtype=object) + for idx, dof in np.ndenumerate(sample_item): + new_arr[idx] = _recursive_resize_reinit_with_zeros( + actx, dof, target_volume_sizes, sample_volume_sizes) + return new_arr + else: + return sample_item # retain non-DOFArray ndarray data outright elif isinstance(sample_item, dict): return {k: _recursive_resize_reinit_with_zeros( actx, v, target_volume_sizes, sample_volume_sizes) @@ -432,7 +443,7 @@ def _extract_src_rank_specific_mapping(trs_olaps, src_rank): def _get_item_structure(item): - """Return a simplified representation of the data structure with field names.""" + """Report data structure with field names.""" if isinstance(item, DOFArray): return "DOFArray" elif isinstance(item, np.ndarray): @@ -457,6 +468,32 @@ def _get_item_structure(item): return type(item).__name__ +def _get_item_structure_and_size(item, ignore_keys=None): + """Report data structure with field names and sizes for DOFArrays.""" + if ignore_keys is None: + ignore_keys = {"volume_to_local_mesh_data"} + if isinstance(item, DOFArray): + shape = item[0].shape + return f"DOFArray({shape[0]} elements, {shape[1]} nodes)" + elif isinstance(item, dict): + return {k: _get_item_structure_and_size(v) + for k, v in item.items() if k not in ignore_keys} + elif isinstance(item, (list, tuple)): + item_structure = [_get_item_structure_and_size(v) for v in item] + return f"{type(item).__name__}({', '.join(map(str, item_structure))})" + elif is_dataclass(item): + item_fields = asdict(item) + field_structure = [f"{k}: {_get_item_structure_and_size(v)}" + for k, v in item_fields.items() if k not in ignore_keys] + return f"{type(item).__name__}({', '.join(field_structure)})" + elif isinstance(item, np.ndarray): + array_structure = [_get_item_structure_and_size(sub_item) + for sub_item in item] + return f"ndarray({', '.join(map(str, array_structure))})" + else: + return type(item).__name__ + + # Call this one with target-rank-specific mappings (trs_olaps) of the form: # {targ_partid : { src_partid : {trg_el_index : src_el_index} } } def _get_restart_data_for_target_rank( From 2e638d3451acac99c97e834616a5fa0d5e7bce20 Mon Sep 17 00:00:00 2001 From: Mike Anderson Date: Fri, 13 Oct 2023 19:26:22 -0500 Subject: [PATCH 41/55] fix meshdist for dimensionality --- bin/meshdist.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bin/meshdist.py b/bin/meshdist.py index 9fe61b10d..7b2180732 100755 --- a/bin/meshdist.py +++ b/bin/meshdist.py @@ -68,7 +68,7 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point -def main(actx_class, mesh_source=None, ndist=None, +def main(actx_class, mesh_source=None, ndist=None, dim=3, output_path=None, log_path=None, casename=None, use_1d_part=None, use_wall=False): """The main function.""" @@ -176,7 +176,7 @@ def main(actx_class, mesh_source=None, ndist=None, def get_mesh_data(): from meshmode.mesh.io import read_gmsh mesh, tag_to_elements = read_gmsh( - mesh_source, + mesh_source, force_ambient_dim=2, return_tag_to_elements_map=True) volume_to_tags = { "fluid": ["fluid"]} @@ -238,6 +238,9 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): action="store_true", help="Include wall domain in mesh.") parser.add_argument("-1", "--1dpart", dest="one_d_part", action="store_true", help="Use 1D partitioner.") + parser.add_argument("-d", "--dimen", type=int, dest="dim", + nargs="?", action="store", + help="Number dimensions") parser.add_argument("-n", "--ndist", type=int, dest="ndist", nargs="?", action="store", help="Number of distributed parts") @@ -257,7 +260,7 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): actx_class = get_reasonable_array_context_class( lazy=False, distributed=True, profiling=False, numpy=False) - main(actx_class, mesh_source=args.source, + main(actx_class, mesh_source=args.source, dim=args.dim, output_path=args.output_path, ndist=args.ndist, log_path=args.log_path, casename=args.casename, use_1d_part=args.one_d_part, use_wall=args.use_wall) From 09339c6133523ea883aadfb6e5d67d0fb1ba4fb9 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Sun, 15 Oct 2023 18:00:35 -0700 Subject: [PATCH 42/55] Add some timings, fix issue with parallel redist. --- mirgecom/restart.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index 4d47ec49b..8e2b211a2 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -32,6 +32,7 @@ import numpy as np from meshmode.dof_array import array_context_for_pickling, DOFArray from grudge.discretization import PartID +from datetime import datetime from dataclasses import is_dataclass, asdict from collections import defaultdict from mirgecom.simutil import ( @@ -620,10 +621,10 @@ def redistribute_multivolume_restart_data( vol_to_sample_mesh_data = \ sample_rst_data.pop(mesh_data_item, None) _ensure_unique_nelems(vol_to_sample_mesh_data) - if my_rank == 0: - with array_context_for_pickling(actx): + with array_context_for_pickling(actx): + inp_rst_structure = _get_item_structure(sample_rst_data) + if my_rank == 0: print("Restart data structure:") - inp_rst_structure = _get_item_structure(sample_rst_data) pprint(inp_rst_structure) # sample_vol_sizes, determine from mesh? @@ -648,15 +649,20 @@ def redistribute_multivolume_restart_data( < nleftover else nleftover) my_ending_rank = my_starting_rank + nparts_this_writer - 1 parts_to_write = list(range(my_starting_rank, my_ending_rank+1)) - print(f"{my_rank=}, {writer_rank=}, {parts_to_write=}") + print(f"{my_rank=}, {writer_rank=}, " + f"Parts[{my_starting_rank},{my_ending_rank}]") # print(f"{source_multivol_decomp_map=}") # print(f"{target_multivol_decomp_map=}") + if writer_rank == 0: + print(f"{datetime.now()}: Computing interdecomp mapping ...") xdo = multivolume_interdecomposition_overlap(source_idecomp_map, target_idecomp_map, source_multivol_decomp_map, target_multivol_decomp_map, return_ranks=parts_to_write) + if writer_rank == 0: + print(f"{datetime.now()}: Computing interdecomp mapping (done)") # for trg_partid, olaps in xdo.items(): # print(f"dbg {trg_partid=}, " @@ -665,6 +671,8 @@ def redistribute_multivolume_restart_data( # print(f"dbg {src_partid=}, {len(elem_map)=}") for trg_rank in parts_to_write: + if writer_rank == 0: + print(f"{datetime.now()}: Processing rank {trg_rank}.") trs_olaps = {k: v for k, v in xdo.items() if k.rank == trg_rank} out_rst_data = _get_restart_data_for_target_rank( actx, trg_rank, sample_rst_data, sample_vol_sizes, trs_olaps, @@ -700,5 +708,9 @@ def redistribute_multivolume_restart_data( with array_context_for_pickling(actx): with open(output_filename, "wb") as f: pickle.dump(out_rst_data, f) + + if writer_rank == 0 and writer_nprocs > 1: + print(f"{datetime.now()}: Waiting on other ranks to finish ...") + writer_comm_wrapper.Barrier() return From 99b400357ac5faa12d039daa153df8295b07c5da Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Sun, 15 Oct 2023 18:04:34 -0700 Subject: [PATCH 43/55] Fix dim parse bug in meshdist. --- bin/meshdist.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/meshdist.py b/bin/meshdist.py index 7b2180732..70150c0fd 100755 --- a/bin/meshdist.py +++ b/bin/meshdist.py @@ -68,7 +68,7 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point -def main(actx_class, mesh_source=None, ndist=None, dim=3, +def main(actx_class, mesh_source=None, ndist=None, dim=None, output_path=None, log_path=None, casename=None, use_1d_part=None, use_wall=False): """The main function.""" @@ -77,6 +77,9 @@ def main(actx_class, mesh_source=None, ndist=None, dim=3, mesh_source.strip("'") + if dim is None: + dim = 3 + if log_path is None: log_path = "log_data" @@ -176,7 +179,7 @@ def main(actx_class, mesh_source=None, ndist=None, dim=3, def get_mesh_data(): from meshmode.mesh.io import read_gmsh mesh, tag_to_elements = read_gmsh( - mesh_source, force_ambient_dim=2, + mesh_source, force_ambient_dim=dim, return_tag_to_elements_map=True) volume_to_tags = { "fluid": ["fluid"]} From 69a6f84272303d25be3a22e46c63283906017c50 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Mon, 16 Oct 2023 10:30:10 -0700 Subject: [PATCH 44/55] Add mapdecomp util. --- bin/mapdecomp.py | 106 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 bin/mapdecomp.py diff --git a/bin/mapdecomp.py b/bin/mapdecomp.py new file mode 100644 index 000000000..348dcaf67 --- /dev/null +++ b/bin/mapdecomp.py @@ -0,0 +1,106 @@ +"""Read gmsh mesh, partition it, and create a pkl file per mesh partition.""" + +__copyright__ = """ +Copyright (C) 2020 University of Illinois Board of Trustees +""" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" +import argparse +import pickle +import os + +from meshmode.distributed import get_connected_parts +from grudge.discretization import PartID + + +def main(mesh_filename=None, output_path=None): + """Do it.""" + if output_path is None: + output_path = "./" + output_path.strip("'") + + if mesh_filename is None: + # Try to detect the mesh filename + raise AssertionError("No mesh filename.") + + intradecomp_map = {} + nranks = 0 + nvolumes = 0 + volumes = set() + for r in range(10000): + mesh_pkl_filename = mesh_filename + f"_rank{r}.pkl" + if os.path.exists(mesh_pkl_filename): + nranks = nranks + 1 + with open(mesh_pkl_filename, "rb") as pkl_file: + global_nelements, volume_to_local_mesh_data = \ + pickle.load(pkl_file) + for vol, meshdat in volume_to_local_mesh_data.items(): + local_partid = PartID(volume_tag=vol, rank=r) + volumes.add(vol) + connected_parts = get_connected_parts(meshdat[0]) + if connected_parts: + intradecomp_map[local_partid] = connected_parts + else: + break + nvolumes = len(volumes) + rank_rank_nbrs = {r: set() for r in range(nranks)} + for part, nbrs in intradecomp_map.items(): + local_rank = part.rank + for nbr in nbrs: + if nbr.rank != local_rank: + rank_rank_nbrs[local_rank].add(nbr.rank) + min_rank_nbrs = nranks + max_rank_nbrs = 0 + num_nbr_dist = {} + total_nnbrs = 0 + for _, rank_nbrs in rank_rank_nbrs.items(): + nrank_nbrs = len(rank_nbrs) + total_nnbrs += nrank_nbrs + if nrank_nbrs not in num_nbr_dist: + num_nbr_dist[nrank_nbrs] = 0 + num_nbr_dist[nrank_nbrs] += 1 + min_rank_nbrs = min(min_rank_nbrs, nrank_nbrs) + max_rank_nbrs = max(max_rank_nbrs, nrank_nbrs) + + mean_nnbrs = (1.0*total_nnbrs) / (1.0*nranks) + + print(f"Number of ranks: {nranks}") + print(f"Number of volumes: {nvolumes}") + print(f"Volumes: {volumes}") + print("Number of rank neighbors (min, max, mean): " + f"({min_rank_nbrs}, {max_rank_nbrs}, {mean_nnbrs})") + print(f"Distribution of num nbrs: {num_nbr_dist=}") + + # print(f"{intradecomp_map=}") + with open(f"intradecomp_map_np{nranks}.pkl", "wb") as pkl_file: + pickle.dump(intradecomp_map, pkl_file) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser( + description="MIRGE-Com Intradecomp mapper") + parser.add_argument("-m", "--mesh", type=str, dest="mesh_filename", + nargs="?", action="store", help="root filename for mesh") + + args = parser.parse_args() + + main(mesh_filename=args.mesh_filename) From a4eed52441e0b598d77423a91b52985f9d7b2f16 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 14 Nov 2023 08:15:24 -0600 Subject: [PATCH 45/55] Merge main --- .github/workflows/ci.yaml | 11 ++- .readthedocs.yml | 7 +- .rtd-env-py3.yml | 5 +- examples/ablation-workshop.py | 10 ++- mirgecom/eos.py | 10 +++ mirgecom/gas_model.py | 44 +++++++++- mirgecom/logging_quantities.py | 1 + mirgecom/mpi.py | 2 +- mirgecom/transport.py | 132 ++++++++++++++++++++++++++++++ scripts/delta-parallel-spawner.sh | 12 +++ scripts/delta.sbatch.sh | 43 ++++++++++ scripts/lassen.bsub.sh | 2 +- scripts/mirge-testing-env.sh | 6 ++ 13 files changed, 273 insertions(+), 12 deletions(-) create mode 100755 scripts/delta-parallel-spawner.sh create mode 100644 scripts/delta.sbatch.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c2d570390..aa0fcc7bf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -119,12 +119,16 @@ jobs: - name: Run examples run: | + set -x MINIFORGE_INSTALL_DIR=.miniforge3 . "$MINIFORGE_INSTALL_DIR/bin/activate" testing export XDG_CACHE_HOME=/tmp mamba install vtk # needed for the accuracy comparison [[ $(hostname) == "porter" ]] && export PYOPENCL_TEST="port:nv" && unset XDG_CACHE_HOME - # && export POCL_DEBUG=cuda + + # This is only possible because actions run sequentially on porter + [[ $(hostname) == "porter" ]] && rm -rf /tmp/githubrunner/pocl-scratch && rm -rf /tmp/githubrunner/xdg-scratch + scripts/run-integrated-tests.sh --examples doc: @@ -195,8 +199,13 @@ jobs: fetch-depth: '0' - name: Prepare production environment run: | + set -x [[ $(uname) == Linux ]] && [[ $(hostname) != "porter" ]] && sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev [[ $(uname) == Darwin ]] && brew upgrade && brew install mpich + + # This is only possible because actions run sequentially on porter + [[ $(hostname) == "porter" ]] && rm -rf /tmp/githubrunner/pocl-scratch && rm -rf /tmp/githubrunner/xdg-scratch + MIRGEDIR=$(pwd) cat scripts/production-testing-env.sh . scripts/production-testing-env.sh diff --git a/.readthedocs.yml b/.readthedocs.yml index 4938b5749..9b31b5b43 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -2,7 +2,12 @@ version: 2 conda: environment: .rtd-env-py3.yml - + +build: + os: "ubuntu-22.04" + tools: + python: "mambaforge-22.9" + sphinx: fail_on_warning: true diff --git a/.rtd-env-py3.yml b/.rtd-env-py3.yml index ee22006c1..4587ba307 100644 --- a/.rtd-env-py3.yml +++ b/.rtd-env-py3.yml @@ -4,13 +4,12 @@ channels: - conda-forge - nodefaults dependencies: -# readthedocs does not yet support Python 3.11 -# See e.g. https://readthedocs.org/api/v2/build/18650881.txt -- python=3.10 +- python=3.11 - mpi4py - islpy - pip - pyopencl +- graphviz - scipy - pip: - "git+https://github.com/inducer/pymbolic.git#egg=pymbolic" diff --git a/examples/ablation-workshop.py b/examples/ablation-workshop.py index 66e465e01..6668745a3 100644 --- a/examples/ablation-workshop.py +++ b/examples/ablation-workshop.py @@ -25,9 +25,9 @@ import logging import gc import numpy as np -import scipy # type: ignore[import] -from scipy.interpolate import CubicSpline # type: ignore[import] -from warnings import warn +import scipy # type: ignore[import-untyped] +from scipy.interpolate import CubicSpline # type: ignore[import-untyped] + from meshmode.dof_array import DOFArray from meshmode.discretization.connection import FACE_RESTR_ALL @@ -488,7 +488,8 @@ def get_species_source_terms(self, cv: ConservedVars, temperature: DOFArray): raise NotImplementedError def dependent_vars(self, cv: ConservedVars, temperature_seed=None, - smoothness_mu=None, smoothness_kappa=None, smoothness_beta=None): + smoothness_mu=None, smoothness_kappa=None, + smoothness_d=None, smoothness_beta=None): raise NotImplementedError @@ -755,6 +756,7 @@ def make_state(cv, temperature_seed, material_densities): smoothness_mu=zeros, smoothness_kappa=zeros, smoothness_beta=zeros, + smoothness_d=zeros, species_enthalpies=cv.species_mass, # empty array ) diff --git a/mirgecom/eos.py b/mirgecom/eos.py index 149e19687..d7de23cee 100644 --- a/mirgecom/eos.py +++ b/mirgecom/eos.py @@ -77,6 +77,7 @@ class GasDependentVars: .. attribute:: smoothness_mu .. attribute:: smoothness_kappa .. attribute:: smoothness_beta + .. attribute:: smoothness_d """ temperature: DOFArray @@ -84,6 +85,7 @@ class GasDependentVars: speed_of_sound: DOFArray smoothness_mu: DOFArray smoothness_kappa: DOFArray + smoothness_d: DOFArray smoothness_beta: DOFArray @@ -179,6 +181,7 @@ def dependent_vars( temperature_seed: Optional[DOFArray] = None, smoothness_mu: Optional[DOFArray] = None, smoothness_kappa: Optional[DOFArray] = None, + smoothness_d: Optional[DOFArray] = None, smoothness_beta: Optional[DOFArray] = None) -> GasDependentVars: """Get an agglomerated array of the dependent variables. @@ -194,6 +197,8 @@ def dependent_vars( smoothness_mu = zeros if smoothness_kappa is None: smoothness_kappa = zeros + if smoothness_d is None: + smoothness_d = zeros if smoothness_beta is None: smoothness_beta = zeros @@ -203,6 +208,7 @@ def dependent_vars( speed_of_sound=self.sound_speed(cv, temperature), smoothness_mu=smoothness_mu, smoothness_kappa=smoothness_kappa, + smoothness_d=smoothness_d, smoothness_beta=smoothness_beta ) @@ -257,6 +263,7 @@ def dependent_vars( temperature_seed: Optional[DOFArray] = None, smoothness_mu: Optional[DOFArray] = None, smoothness_kappa: Optional[DOFArray] = None, + smoothness_d: Optional[DOFArray] = None, smoothness_beta: Optional[DOFArray] = None) -> MixtureDependentVars: """Get an agglomerated array of the dependent variables. @@ -272,6 +279,8 @@ def dependent_vars( smoothness_mu = zeros if smoothness_kappa is None: smoothness_kappa = zeros + if smoothness_d is None: + smoothness_d = zeros if smoothness_beta is None: smoothness_beta = zeros @@ -282,6 +291,7 @@ def dependent_vars( species_enthalpies=self.species_enthalpies(cv, temperature), smoothness_mu=smoothness_mu, smoothness_kappa=smoothness_kappa, + smoothness_d=smoothness_d, smoothness_beta=smoothness_beta ) diff --git a/mirgecom/gas_model.py b/mirgecom/gas_model.py index 4791270a7..46563d2bc 100644 --- a/mirgecom/gas_model.py +++ b/mirgecom/gas_model.py @@ -118,6 +118,7 @@ class FluidState: .. autoattribute:: temperature .. autoattribute:: smoothness_mu .. autoattribute:: smoothness_kappa + .. autoattribute:: smoothness_d .. autoattribute:: smoothness_beta .. autoattribute:: velocity .. autoattribute:: speed @@ -169,6 +170,11 @@ def smoothness_kappa(self): """Return the smoothness_kappa field.""" return self.dv.smoothness_kappa + @property + def smoothness_d(self): + """Return the smoothness_d field.""" + return self.dv.smoothness_d + @property def smoothness_beta(self): """Return the smoothness_beta field.""" @@ -297,6 +303,7 @@ def make_fluid_state(cv, gas_model, temperature_seed=None, smoothness_mu=None, smoothness_kappa=None, + smoothness_d=None, smoothness_beta=None, material_densities=None, limiter_func=None, limiter_dd=None): @@ -328,6 +335,11 @@ def make_fluid_state(cv, gas_model, Optional array containing the smoothness parameter for extra thermal conductivity in the artificial viscosity. + smoothness_d: :class:`~meshmode.dof_array.DOFArray` + + Optional array containing the smoothness parameter for extra species + diffusivity in the artificial viscosity. + smoothness_beta: :class:`~meshmode.dof_array.DOFArray` Optional array containing the smoothness parameter for extra bulk @@ -357,6 +369,8 @@ def make_fluid_state(cv, gas_model, is None else smoothness_kappa) smoothness_beta = (actx.np.zeros_like(cv.mass) if smoothness_beta is None else smoothness_beta) + smoothness_d = (actx.np.zeros_like(cv.mass) if smoothness_d + is None else smoothness_d) if isinstance(gas_model, GasModel): temperature = gas_model.eos.temperature(cv=cv, @@ -373,6 +387,7 @@ def make_fluid_state(cv, gas_model, speed_of_sound=gas_model.eos.sound_speed(cv, temperature), smoothness_mu=smoothness_mu, smoothness_kappa=smoothness_kappa, + smoothness_d=smoothness_d, smoothness_beta=smoothness_beta ) @@ -384,6 +399,7 @@ def make_fluid_state(cv, gas_model, speed_of_sound=dv.speed_of_sound, smoothness_mu=dv.smoothness_mu, smoothness_kappa=dv.smoothness_kappa, + smoothness_d=dv.smoothness_d, smoothness_beta=dv.smoothness_beta, species_enthalpies=gas_model.eos.species_enthalpies(cv, temperature) ) @@ -428,6 +444,7 @@ def make_fluid_state(cv, gas_model, speed_of_sound=gas_model.eos.sound_speed(cv, temperature), smoothness_mu=smoothness_mu, smoothness_kappa=smoothness_kappa, + smoothness_d=smoothness_d, smoothness_beta=smoothness_beta, species_enthalpies=gas_model.eos.species_enthalpies(cv, temperature), ) @@ -507,6 +524,10 @@ def project_fluid_state(dcoll, src, tgt, state, gas_model, limiter_func=None, if state.dv.smoothness_kappa is not None: smoothness_kappa = op.project(dcoll, src, tgt, state.dv.smoothness_kappa) + smoothness_d = None + if state.dv.smoothness_d is not None: + smoothness_d = op.project(dcoll, src, tgt, state.dv.smoothness_d) + smoothness_beta = None if state.dv.smoothness_beta is not None: smoothness_beta = op.project(dcoll, src, tgt, state.dv.smoothness_beta) @@ -519,6 +540,7 @@ def project_fluid_state(dcoll, src, tgt, state, gas_model, limiter_func=None, temperature_seed=temperature_seed, smoothness_mu=smoothness_mu, smoothness_kappa=smoothness_kappa, + smoothness_d=smoothness_d, smoothness_beta=smoothness_beta, material_densities=material_densities, limiter_func=limiter_func, limiter_dd=tgt) @@ -535,6 +557,7 @@ def make_fluid_state_trace_pairs(cv_pairs, gas_model, temperature_seed_pairs=None, smoothness_mu_pairs=None, smoothness_kappa_pairs=None, + smoothness_d_pairs=None, smoothness_beta_pairs=None, material_densities_pairs=None, limiter_func=None): @@ -579,6 +602,8 @@ def make_fluid_state_trace_pairs(cv_pairs, gas_model, smoothness_mu_pairs = [None] * len(cv_pairs) if smoothness_kappa_pairs is None: smoothness_kappa_pairs = [None] * len(cv_pairs) + if smoothness_d_pairs is None: + smoothness_d_pairs = [None] * len(cv_pairs) if smoothness_beta_pairs is None: smoothness_beta_pairs = [None] * len(cv_pairs) if material_densities_pairs is None: @@ -590,6 +615,7 @@ def make_fluid_state_trace_pairs(cv_pairs, gas_model, temperature_seed=_getattr_ish(tseed_pair, "int"), smoothness_mu=_getattr_ish(smoothness_mu_pair, "int"), smoothness_kappa=_getattr_ish(smoothness_kappa_pair, "int"), + smoothness_d=_getattr_ish(smoothness_d_pair, "int"), smoothness_beta=_getattr_ish(smoothness_beta_pair, "int"), material_densities=_getattr_ish(material_densities_pair, "int"), limiter_func=limiter_func, limiter_dd=cv_pair.dd), @@ -598,6 +624,7 @@ def make_fluid_state_trace_pairs(cv_pairs, gas_model, temperature_seed=_getattr_ish(tseed_pair, "ext"), smoothness_mu=_getattr_ish(smoothness_mu_pair, "ext"), smoothness_kappa=_getattr_ish(smoothness_kappa_pair, "ext"), + smoothness_d=_getattr_ish(smoothness_d_pair, "ext"), smoothness_beta=_getattr_ish(smoothness_beta_pair, "ext"), material_densities=_getattr_ish(material_densities_pair, "ext"), limiter_func=limiter_func, limiter_dd=cv_pair.dd)) @@ -605,10 +632,11 @@ def make_fluid_state_trace_pairs(cv_pairs, gas_model, tseed_pair, smoothness_mu_pair, smoothness_kappa_pair, + smoothness_d_pair, smoothness_beta_pair, material_densities_pair in zip( cv_pairs, temperature_seed_pairs, - smoothness_mu_pairs, smoothness_kappa_pairs, + smoothness_mu_pairs, smoothness_kappa_pairs, smoothness_d_pairs, smoothness_beta_pairs, material_densities_pairs)] @@ -628,6 +656,10 @@ class _FluidSmoothnessKappaTag: pass +class _FluidSmoothnessDiffTag: + pass + + class _FluidSmoothnessBetaTag: pass @@ -758,6 +790,14 @@ def make_operator_fluid_states( dcoll, volume_state.smoothness_kappa, volume_dd=dd_vol, tag=(_FluidSmoothnessKappaTag, comm_tag))] + smoothness_d_interior_pairs = None + if volume_state.smoothness_d is not None: + smoothness_d_interior_pairs = [ + interp_to_surf_quad(tpair=tpair) + for tpair in interior_trace_pairs( + dcoll, volume_state.smoothness_d, volume_dd=dd_vol, + tag=(_FluidSmoothnessDiffTag, comm_tag))] + smoothness_beta_interior_pairs = None if volume_state.smoothness_beta is not None: smoothness_beta_interior_pairs = [ @@ -780,6 +820,7 @@ def make_operator_fluid_states( temperature_seed_pairs=tseed_interior_pairs, smoothness_mu_pairs=smoothness_mu_interior_pairs, smoothness_kappa_pairs=smoothness_kappa_interior_pairs, + smoothness_d_pairs=smoothness_d_interior_pairs, smoothness_beta_pairs=smoothness_beta_interior_pairs, material_densities_pairs=material_densities_interior_pairs, limiter_func=limiter_func) @@ -874,6 +915,7 @@ def replace_fluid_state( temperature_seed=new_tseed, smoothness_mu=state.smoothness_mu, smoothness_kappa=state.smoothness_kappa, + smoothness_d=state.smoothness_d, smoothness_beta=state.smoothness_beta, material_densities=material_densities, limiter_func=limiter_func, diff --git a/mirgecom/logging_quantities.py b/mirgecom/logging_quantities.py index 6271ce620..0e68e3048 100644 --- a/mirgecom/logging_quantities.py +++ b/mirgecom/logging_quantities.py @@ -69,6 +69,7 @@ def initialize_logmgr(enable_logmgr: bool, logmgr = LogManager(filename=filename, mode=mode, mpi_comm=mpi_comm) logmgr.add_quantity(PythonInitTime()) + logmgr.enable_save_on_sigterm() add_run_info(logmgr) add_package_versions(logmgr) diff --git a/mirgecom/mpi.py b/mirgecom/mpi.py index 69865b8bd..cf9641a87 100644 --- a/mirgecom/mpi.py +++ b/mirgecom/mpi.py @@ -171,7 +171,7 @@ def _check_isl_version() -> None: library to determine if we are running with imath-32. """ import ctypes - import islpy # type: ignore[import] + import islpy # type: ignore[import-untyped] try: ctypes.cdll.LoadLibrary(islpy._isl.__file__).isl_val_get_num_gmp diff --git a/mirgecom/transport.py b/mirgecom/transport.py index 4f6e5bf6f..9260488fb 100644 --- a/mirgecom/transport.py +++ b/mirgecom/transport.py @@ -16,6 +16,7 @@ .. autoclass:: MixtureAveragedTransport .. autoclass:: ArtificialViscosityTransportDiv .. autoclass:: ArtificialViscosityTransportDiv2 +.. autoclass:: ArtificialViscosityTransportDiv3 Exceptions ^^^^^^^^^^ @@ -698,3 +699,134 @@ def species_diffusivity(self, cv: ConservedVars, eos: Optional[GasEOS] = None) -> DOFArray: r"""Get the vector of species diffusivities, ${d}_{\alpha}$.""" return self._physical_transport.species_diffusivity(cv, dv, eos) + + +class ArtificialViscosityTransportDiv3(TransportModel): + r"""Transport model for add artificial viscosity. + + Inherits from (and implements) :class:`TransportModel`. + + Takes a physical transport model and adds the artificial viscosity + contribution to it. Defaults to simple transport with inviscid settings. + This is equivalent to inviscid flow with artifical viscosity enabled. + + .. automethod:: __init__ + .. automethod:: bulk_viscosity + .. automethod:: viscosity + .. automethod:: volume_viscosity + .. automethod:: thermal_conductivity + .. automethod:: species_diffusivity + """ + + def __init__(self, + av_mu, av_kappa, av_beta, av_d, av_prandtl, + physical_transport=None, + av_species_diffusivity=None): + """Initialize uniform, constant transport properties.""" + if physical_transport is None: + self._physical_transport = SimpleTransport() + else: + self._physical_transport = physical_transport + + if av_species_diffusivity is None: + av_species_diffusivity = np.empty((0,), dtype=object) + + self._av_mu = av_mu + self._av_beta = av_beta + self._av_kappa = av_kappa + self._av_d = av_d + self._av_prandtl = av_prandtl + + def av_mu(self, cv, dv, eos): + r"""Get the shear artificial viscosity for the gas.""" + actx = cv.array_context + return (self._av_mu * cv.mass + * actx.np.sqrt(np.dot(cv.velocity, cv.velocity) + + dv.speed_of_sound**2)) + + def av_beta(self, cv, dv, eos): + r"""Get the shear artificial viscosity for the gas.""" + actx = cv.array_context + return (self._av_beta * cv.mass + * actx.np.sqrt(np.dot(cv.velocity, cv.velocity) + + dv.speed_of_sound**2)) + + def av_kappa(self, cv, dv, eos): + r"""Get the shear artificial viscosity for the gas.""" + actx = cv.array_context + return (self._av_kappa * cv.mass + * actx.np.sqrt(np.dot(cv.velocity, cv.velocity) + + dv.speed_of_sound**2)) + + def av_d(self, cv, dv, eos): + r"""Get the shear artificial viscosity for the gas.""" + actx = cv.array_context + return (self._av_d * actx.np.sqrt(np.dot(cv.velocity, cv.velocity) + + dv.speed_of_sound**2)) + + def bulk_viscosity(self, cv: ConservedVars, # type: ignore[override] + dv: GasDependentVars, + eos: GasEOS) -> DOFArray: + r"""Get the bulk viscosity for the gas, $\mu_{B}$.""" + actx = cv.array_context + smoothness_beta = actx.np.where( + actx.np.greater(dv.smoothness_beta, 0.), dv.smoothness_beta, 0.) + return (smoothness_beta*self.av_beta(cv, dv, eos) + + self._physical_transport.bulk_viscosity(cv, dv)) + + def viscosity(self, cv: ConservedVars, # type: ignore[override] + dv: GasDependentVars, + eos: GasEOS) -> DOFArray: + r"""Get the gas dynamic viscosity, $\mu$.""" + actx = cv.array_context + smoothness_mu = actx.np.where( + actx.np.greater(dv.smoothness_mu, 0.), dv.smoothness_mu, 0.) + return (smoothness_mu*self.av_mu(cv, dv, eos) + + self._physical_transport.viscosity(cv, dv)) + + def volume_viscosity(self, cv: ConservedVars, # type: ignore[override] + dv: GasDependentVars, + eos: GasEOS) -> DOFArray: + r"""Get the 2nd viscosity coefficent, $\lambda$. + + In this transport model, the second coefficient of viscosity is: + + $\lambda = \left(\mu_{B} - \frac{2\mu}{3}\right)$ + """ + actx = cv.array_context + smoothness_mu = actx.np.where( + actx.np.greater(dv.smoothness_mu, 0.), dv.smoothness_mu, 0.) + smoothness_beta = actx.np.where( + actx.np.greater(dv.smoothness_beta, 0.), dv.smoothness_beta, 0.) + + return (smoothness_beta*self.av_beta(cv, dv, eos) + - 2*smoothness_mu*self.av_mu(cv, dv, eos)/3 + + self._physical_transport.volume_viscosity(cv, dv)) + + def thermal_conductivity(self, cv: ConservedVars, # type: ignore[override] + dv: GasDependentVars, + eos: GasEOS) -> DOFArray: + r"""Get the gas thermal_conductivity, $\kappa$.""" + cp = eos.heat_capacity_cp(cv, dv.temperature) + actx = cv.array_context + smoothness_beta = actx.np.where( + actx.np.greater(dv.smoothness_beta, 0.), dv.smoothness_beta, 0.) + smoothness_kappa = actx.np.where( + actx.np.greater(dv.smoothness_kappa, 0.), dv.smoothness_kappa, 0.) + av_kappa = ( + cp*(smoothness_beta*self.av_beta(cv, dv, eos)/self._av_prandtl + + smoothness_kappa*self.av_kappa(cv, dv, eos)) + ) + + return (av_kappa + + self._physical_transport.thermal_conductivity(cv, dv, eos)) + + def species_diffusivity(self, cv: ConservedVars, # type: ignore[override] + dv: GasDependentVars, + eos: GasEOS) -> DOFArray: + r"""Get the vector of species diffusivities, ${d}_{\alpha}$.""" + actx = cv.array_context + smoothness_d = actx.np.where( + actx.np.greater(dv.smoothness_d, 0.), dv.smoothness_d, 0.) + return (smoothness_d*self.av_d(cv, dv, eos) + + self._physical_transport.species_diffusivity(cv, dv, eos)) diff --git a/scripts/delta-parallel-spawner.sh b/scripts/delta-parallel-spawner.sh new file mode 100755 index 000000000..9a3f62757 --- /dev/null +++ b/scripts/delta-parallel-spawner.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Used to wrap the spawning of parallel mirgecom drivers on Delta. +# unset CUDA_CACHE_DISABLE +POCL_CACHE_ROOT=${POCL_CACHE_ROOT:-"/tmp/$USER/pocl-scratch"} +XDG_CACHE_ROOT=${XDG_CACHE_HOME:-"/tmp/$USER/xdg-scratch"} +POCL_CACHE_DIR=${POCL_CACHE_DIR:-"${POCL_CACHE_ROOT}/rank$SLURM_PROCID"} +XDG_CACHE_HOME=${XDG_CACHE_HOME:-"${XDG_CACHE_ROOT}/rank$SLURM_PROCID"} +export POCL_CACHE_DIR +export XDG_CACHE_HOME + +"$@" diff --git a/scripts/delta.sbatch.sh b/scripts/delta.sbatch.sh new file mode 100644 index 000000000..b5069407f --- /dev/null +++ b/scripts/delta.sbatch.sh @@ -0,0 +1,43 @@ +#!/bin/bash +#SBATCH --nodes=1 # number of nodes +#SBATCH -t 00:30:00 # walltime (hh:mm:ss) +#SBATCH --partition=gpuA40x4 +#SBATCH --ntasks-per-node=4 # change this if running on a partition with other than 4 GPUs per node +#SBATCH --gpus-per-node=4 # change this if running on a partition with other than 4 GPUs per node +#SBATCH --gpu-bind=single:1 +#SBATCH --account=bbkf-delta-gpu +#SBATCH --exclusive # dedicated node for this job +#SBATCH --no-requeue +#SBATCH --gpus-per-task=1 + +# Run this script with 'sbatch delta.sbatch.sh' + +# Delta user guide: +# - https://wiki.ncsa.illinois.edu/display/DSC/Delta+User+Guide +# - https://ncsa-delta-doc.readthedocs-hosted.com/en/latest/ + +# Put any environment activation here, e.g.: +# source ../../config/activate_env.sh + +# OpenCL device selection: +export PYOPENCL_CTX="port:nvidia" # Run on Nvidia GPU with pocl +# export PYOPENCL_CTX="port:pthread" # Run on CPU with pocl + +nnodes=$SLURM_JOB_NUM_NODES +nproc=$SLURM_NTASKS + +echo nnodes=$nnodes nproc=$nproc + +srun_cmd="srun -N $nnodes -n $nproc" + +# See +# https://mirgecom.readthedocs.io/en/latest/running.html#avoiding-overheads-due-to-caching-of-kernels +# on why this is important +export XDG_CACHE_HOME_ROOT="/tmp/$USER/xdg-scratch/rank" + +# Fixes https://github.com/illinois-ceesd/mirgecom/issues/292 +# (each rank needs its own POCL cache dir) +export POCL_CACHE_DIR_ROOT="/tmp/$USER/pocl-cache/rank" + +# Run application +$srun_cmd bash -c 'POCL_CACHE_DIR=$POCL_CACHE_DIR_ROOT$SLURM_PROCID XDG_CACHE_HOME=$XDG_CACHE_HOME_ROOT$SLURM_PROCID python -u -O -m mpi4py ./pulse.py' diff --git a/scripts/lassen.bsub.sh b/scripts/lassen.bsub.sh index 9399a9e10..58e602826 100755 --- a/scripts/lassen.bsub.sh +++ b/scripts/lassen.bsub.sh @@ -41,4 +41,4 @@ echo "----------------------------" # -O: switch on optimizations # POCL_CACHE_DIR=...: each rank needs its own POCL cache dir # XDG_CACHE_HOME=...: each rank needs its own Loopy/PyOpenCL cache dir -$jsrun_cmd bash -c 'POCL_CACHE_DIR=$POCL_CACHE_DIR_ROOT$OMPI_COMM_WORLD_RANK XDG_CACHE_HOME=XDG_CACHE_HOME_ROOT$OMPI_COMM_WORLD_RANK python -O -m mpi4py ./pulse-mpi.py' +$jsrun_cmd bash -c 'POCL_CACHE_DIR=$POCL_CACHE_DIR_ROOT$OMPI_COMM_WORLD_RANK XDG_CACHE_HOME=$XDG_CACHE_HOME_ROOT$OMPI_COMM_WORLD_RANK python -O -m mpi4py ./pulse-mpi.py' diff --git a/scripts/mirge-testing-env.sh b/scripts/mirge-testing-env.sh index 017f30ed1..fce711687 100755 --- a/scripts/mirge-testing-env.sh +++ b/scripts/mirge-testing-env.sh @@ -32,6 +32,12 @@ elif [[ $(hostname) == "lassen"* ]]; then PYOPENCL_TEST="port:tesla" PYOPENCL_CTX="port:tesla" MIRGE_MPI_EXEC="jsrun -g 1 -a 1" + +elif [[ $(hostname) == "delta"* ]]; then + MIRGE_PARALLEL_SPAWNER="bash ${MIRGE_HOME}/scripts/delta-parallel-spawner.sh" + PYOPENCL_CTX="port:nvidia" + PYOPENCL_TEST="port:nvidia" + MIRGE_MPI_EXEC="srun" fi export MIRGE_HOME From dd8f535f0bc84de7293796b0a6637a834807f5b8 Mon Sep 17 00:00:00 2001 From: "Michael T. Campbell" Date: Wed, 21 Aug 2024 20:25:49 -0700 Subject: [PATCH 46/55] Allow bigger partition discrepancies, include missing import --- bin/meshdist.py | 3 ++- mirgecom/simutil.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/bin/meshdist.py b/bin/meshdist.py index 70150c0fd..d4b3b4214 100755 --- a/bin/meshdist.py +++ b/bin/meshdist.py @@ -195,7 +195,8 @@ def get_mesh_data(): def my_partitioner(mesh, tag_to_elements, num_ranks): from mirgecom.simutil import geometric_mesh_partitioner return geometric_mesh_partitioner( - mesh, num_ranks, auto_balance=True, debug=False) + mesh, num_ranks, auto_balance=True, debug=True, + imbalance_tolerance=.2) part_func = my_partitioner if use_1d_part else None diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index f3062362a..56414af51 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -94,7 +94,10 @@ DiscretizationDOFAxisTag ) from arraycontext import flatten, map_array_container -from grudge.discretization import DiscretizationCollection +from grudge.discretization import ( + DiscretizationCollection, + PartID +) from grudge.dof_desc import DD_VOLUME_ALL from meshmode.dof_array import DOFArray from collections import defaultdict From ef13518af25252eef4e8f0ea6cac1a0ddf480a78 Mon Sep 17 00:00:00 2001 From: Mike Anderson Date: Tue, 27 Aug 2024 13:40:30 -0700 Subject: [PATCH 47/55] set imbalance tolerance from the outside --- bin/meshdist.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bin/meshdist.py b/bin/meshdist.py index d4b3b4214..581fbd19e 100755 --- a/bin/meshdist.py +++ b/bin/meshdist.py @@ -70,7 +70,8 @@ class MyRuntimeError(RuntimeError): @mpi_entry_point def main(actx_class, mesh_source=None, ndist=None, dim=None, output_path=None, log_path=None, - casename=None, use_1d_part=None, use_wall=False): + casename=None, use_1d_part=None, use_wall=False, + imba_tol=0.01): """The main function.""" if mesh_source is None: raise ApplicationOptionsError("Missing mesh source file.") @@ -196,7 +197,7 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): from mirgecom.simutil import geometric_mesh_partitioner return geometric_mesh_partitioner( mesh, num_ranks, auto_balance=True, debug=True, - imbalance_tolerance=.2) + imbalance_tolerance=imba_tol) part_func = my_partitioner if use_1d_part else None @@ -257,6 +258,8 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): action="store", help="Root name of distributed mesh pkl files.") parser.add_argument("-g", "--logpath", type=str, dest="log_path", nargs="?", action="store", help="simulation case name") + parser.add_argument("-z", "--imbatol", type=float, dest="imbalance_tolerance", nargs="?", + action="store", help="1d partioner imabalance tolerance") args = parser.parse_args() @@ -267,4 +270,5 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): main(actx_class, mesh_source=args.source, dim=args.dim, output_path=args.output_path, ndist=args.ndist, log_path=args.log_path, casename=args.casename, - use_1d_part=args.one_d_part, use_wall=args.use_wall) + use_1d_part=args.one_d_part, use_wall=args.use_wall, + imba_tol=imbalance_tolerance) From 9059d18692af598807c36de77b7590a843e9907f Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 27 Aug 2024 18:56:55 -0500 Subject: [PATCH 48/55] Update to remove mpi_distribute --- mirgecom/gas_model.py | 8 -------- mirgecom/simutil.py | 34 +++++++++++++++++----------------- 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/mirgecom/gas_model.py b/mirgecom/gas_model.py index 26ec36800..d7dfe273d 100644 --- a/mirgecom/gas_model.py +++ b/mirgecom/gas_model.py @@ -806,14 +806,6 @@ def make_operator_fluid_states( dcoll, volume_state.smoothness_d, volume_dd=dd_vol, comm_tag=(_FluidSmoothnessDiffTag, comm_tag))] - smoothness_d_interior_pairs = None - if volume_state.smoothness_d is not None: - smoothness_d_interior_pairs = [ - interp_to_surf_quad(tpair=tpair) - for tpair in interior_trace_pairs( - dcoll, volume_state.smoothness_d, volume_dd=dd_vol, - tag=(_FluidSmoothnessDiffTag, comm_tag))] - smoothness_beta_interior_pairs = None if volume_state.smoothness_beta is not None: smoothness_beta_interior_pairs = [ diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 56414af51..35b5baf7c 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -76,11 +76,10 @@ THE SOFTWARE. """ import logging -import sys import os import pickle from functools import partial -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import Dict, List, Optional from contextlib import contextmanager from logpyle import IntervalTimer @@ -1062,7 +1061,6 @@ def distribute_mesh(comm, get_mesh_data, partition_generator_func=None, logmgr=N from mpi4py import MPI from mpi4py.util import pkl5 from socket import gethostname - from meshmode.distributed import mpi_distribute num_ranks = comm.Get_size() my_global_rank = comm.Get_rank() @@ -1088,6 +1086,7 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks): reader_color = 0 if my_node_rank == 0 else 1 reader_comm = comm.Split(reader_color, my_global_rank) my_reader_rank = reader_comm.Get_rank() + num_node_ranks = node_comm.Get_size() if my_node_rank == 0: num_reading_ranks = reader_comm.Get_size() @@ -1146,7 +1145,7 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks): partition_generator_func(mesh, tag_to_elements, num_ranks) - def get_rank_to_mesh_data(): + def get_rank_to_mesh_data_dict(): if tag_to_elements is None: rank_to_mesh_data = _partition_single_volume_mesh( mesh, num_ranks, rank_per_element, @@ -1163,11 +1162,11 @@ def get_rank_to_mesh_data(): rank: node_rank for node_rank, rank in enumerate(node_ranks)} - node_rank_to_mesh_data = { + node_rank_to_mesh_data_dict = { rank_to_node_rank[rank]: mesh_data for rank, mesh_data in rank_to_mesh_data.items()} - return node_rank_to_mesh_data + return node_rank_to_mesh_data_dict reader_comm.Barrier() if my_reader_rank == 0: @@ -1176,9 +1175,13 @@ def get_rank_to_mesh_data(): if logmgr: logmgr.add_quantity(t_mesh_split) with t_mesh_split.get_sub_timer(): - node_rank_to_mesh_data = get_rank_to_mesh_data() + node_rank_to_mesh_data_dict = get_rank_to_mesh_data_dict() else: - node_rank_to_mesh_data = get_rank_to_mesh_data() + node_rank_to_mesh_data_dict = get_rank_to_mesh_data_dict() + + node_rank_to_mesh_data = [ + node_rank_to_mesh_data_dict[rank] + for rank in range(num_node_ranks)] reader_comm.Barrier() if my_reader_rank == 0: @@ -1189,13 +1192,11 @@ def get_rank_to_mesh_data(): if logmgr: logmgr.add_quantity(t_mesh_dist) with t_mesh_dist.get_sub_timer(): - local_mesh_data = mpi_distribute( - node_comm, source_rank=0, - source_data=node_rank_to_mesh_data) + local_mesh_data = \ + node_comm.scatter(node_rank_to_mesh_data, root=0) else: - local_mesh_data = mpi_distribute( - node_comm, source_rank=0, - source_data=node_rank_to_mesh_data) + local_mesh_data = \ + node_comm.scatter(node_rank_to_mesh_data, root=0) else: # my_node_rank > 0, get mesh part from MPI global_nelements = node_comm.bcast(None, root=0) @@ -1203,10 +1204,9 @@ def get_rank_to_mesh_data(): if logmgr: logmgr.add_quantity(t_mesh_dist) with t_mesh_dist.get_sub_timer(): - local_mesh_data = \ - mpi_distribute(node_comm, source_rank=0) + local_mesh_data = node_comm.scatter(None, root=0) else: - local_mesh_data = mpi_distribute(node_comm, source_rank=0) + local_mesh_data = node_comm.scatter(None, root=0) return local_mesh_data, global_nelements From beca614db5b1755f4bb9a96bc4501bb17d42c1ad Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 27 Aug 2024 19:12:55 -0500 Subject: [PATCH 49/55] Fix up some linting errors --- bin/meshdist.py | 21 ++++++++------------- mirgecom/restart.py | 2 +- mirgecom/simutil.py | 1 + test/test_restart.py | 23 +++++++++++------------ 4 files changed, 21 insertions(+), 26 deletions(-) diff --git a/bin/meshdist.py b/bin/meshdist.py index 581fbd19e..f09bf0683 100755 --- a/bin/meshdist.py +++ b/bin/meshdist.py @@ -28,22 +28,17 @@ import sys import os -from pytools.obj_array import make_obj_array -from functools import partial - -from logpyle import IntervalTimer, set_dt +from logpyle import set_dt from mirgecom.logging_quantities import ( initialize_logmgr, logmgr_add_cl_device_info, logmgr_set_time, - logmgr_add_simulation_info, logmgr_add_device_memory_usage, logmgr_add_mempool_usage, ) from mirgecom.simutil import ( ApplicationOptionsError, - distribute_mesh, distribute_mesh_pkl ) from mirgecom.mpi import mpi_entry_point @@ -141,14 +136,12 @@ def main(actx_class, mesh_source=None, ndist=None, dim=None, logmgr = initialize_logmgr(True, filename=logname, mode="wu", mpi_comm=comm) - from mirgecom.array_context import initialize_actx, actx_class_is_profiling + from mirgecom.array_context import initialize_actx actx = initialize_actx(actx_class, comm) queue = getattr(actx, "queue", None) - use_profiling = actx_class_is_profiling(actx_class) alloc = getattr(actx, "allocator", None) monitor_memory = True - monitor_performance = 2 logmgr_add_cl_device_info(logmgr, queue) @@ -255,11 +248,13 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): nargs="?", action="store", help="Output path for distributed mesh pkl files") parser.add_argument("-c", "--casename", type=str, dest="casename", nargs="?", - action="store", help="Root name of distributed mesh pkl files.") + action="store", + help="Root name of distributed mesh pkl files.") parser.add_argument("-g", "--logpath", type=str, dest="log_path", nargs="?", action="store", help="simulation case name") - parser.add_argument("-z", "--imbatol", type=float, dest="imbalance_tolerance", nargs="?", - action="store", help="1d partioner imabalance tolerance") + parser.add_argument("-z", "--imbatol", type=float, dest="imbalance_tolerance", + nargs="?", action="store", + help="1d partioner imabalance tolerance") args = parser.parse_args() @@ -271,4 +266,4 @@ def my_partitioner(mesh, tag_to_elements, num_ranks): output_path=args.output_path, ndist=args.ndist, log_path=args.log_path, casename=args.casename, use_1d_part=args.one_d_part, use_wall=args.use_wall, - imba_tol=imbalance_tolerance) + imba_tol=args.imbalance_tolerance) diff --git a/mirgecom/restart.py b/mirgecom/restart.py index 8e2b211a2..ee3f73700 100644 --- a/mirgecom/restart.py +++ b/mirgecom/restart.py @@ -708,7 +708,7 @@ def redistribute_multivolume_restart_data( with array_context_for_pickling(actx): with open(output_filename, "wb") as f: pickle.dump(out_rst_data, f) - + if writer_rank == 0 and writer_nprocs > 1: print(f"{datetime.now()}: Waiting on other ranks to finish ...") writer_comm_wrapper.Barrier() diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 35b5baf7c..15566f97e 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -1097,6 +1097,7 @@ def partition_generator_func(mesh, tag_to_elements, num_ranks): print(f"Read(rank, batch): Dist({my_reader_rank}, " f"{read_batch}) on {hostname}.") + global_data = None if logmgr: logmgr.add_quantity(t_mesh_data) with t_mesh_data.get_sub_timer(): diff --git a/test/test_restart.py b/test/test_restart.py index c284fad10..47bbef507 100644 --- a/test/test_restart.py +++ b/test/test_restart.py @@ -279,25 +279,24 @@ def test_multivolume_interdecomp_overlap_basic(): for trg_partid, src_partid_mappings in mv_idx.items(): mapped_trg_elems = set() # validate ranks in trg mapping - assert 0 <= trg_partid.rank < trg_np,\ - f"Invalid target rank: {trg_partid.rank}" + assert 0 <= trg_partid.rank < trg_np, f"Bad target rank: {trg_partid.rank}" for src_partid, element_mapping in src_partid_mappings.items(): # check for consistent volume_tags assert trg_partid.volume_tag == src_partid.volume_tag, \ f"Volume tag mismatch: {trg_partid.volume_tag} "\ f"vs {src_partid.volume_tag}" # validate ranks in src mapping - assert 0 <= src_partid.rank < src_np,\ - f"Invalid source rank: {src_partid.rank}" + assert 0 <= src_partid.rank < src_np, \ + f"Bad source rank:{src_partid.rank}" for trg_local_idx, src_local_idx in element_mapping.items(): # check that each trg el is mapped only once - assert trg_local_idx not in mapped_trg_elems,\ + assert trg_local_idx not in mapped_trg_elems, \ f"Duplicate mapping for target element {trg_local_idx}" mapped_trg_elems.add(trg_local_idx) # check for valid src and trg indices - assert 0 <= src_local_idx < len(src_vol_decomp[src_partid]),\ + assert 0 <= src_local_idx < len(src_vol_decomp[src_partid]), \ f"Invalid source index {src_local_idx} for {src_partid}" - assert 0 <= trg_local_idx < len(trg_vol_decomp[trg_partid]),\ + assert 0 <= trg_local_idx < len(trg_vol_decomp[trg_partid]), \ f"Invalid target index {trg_local_idx} for {trg_partid}" # Check that the mapping is 1-to-1, that each src element is covered and maps @@ -403,7 +402,7 @@ def test_multivolume_interdecomp_overlap(decomp_pattern, vol_pattern, src_trg_ra for trg_partid, src_partid_mappings in mv_idx.items(): mapped_trg_elems = set() # validate ranks in trg mapping - assert 0 <= trg_partid.rank < trg_np,\ + assert 0 <= trg_partid.rank < trg_np, \ f"Invalid target rank: {trg_partid.rank}" for src_partid, element_mapping in src_partid_mappings.items(): # check for consistent volume_tags @@ -411,17 +410,17 @@ def test_multivolume_interdecomp_overlap(decomp_pattern, vol_pattern, src_trg_ra f"Volume tag mismatch: {trg_partid.volume_tag} "\ f"vs {src_partid.volume_tag}" # validate ranks in src mapping - assert 0 <= src_partid.rank < src_np,\ + assert 0 <= src_partid.rank < src_np, \ f"Invalid source rank: {src_partid.rank}" for trg_local_idx, src_local_idx in element_mapping.items(): # check that each trg el is mapped only once - assert trg_local_idx not in mapped_trg_elems,\ + assert trg_local_idx not in mapped_trg_elems, \ f"Duplicate mapping for target element {trg_local_idx}" mapped_trg_elems.add(trg_local_idx) # check for valid src and trg indices - assert 0 <= src_local_idx < len(src_vol_decomp[src_partid]),\ + assert 0 <= src_local_idx < len(src_vol_decomp[src_partid]), \ f"Invalid source index {src_local_idx} for {src_partid}" - assert 0 <= trg_local_idx < len(trg_vol_decomp[trg_partid]),\ + assert 0 <= trg_local_idx < len(trg_vol_decomp[trg_partid]), \ f"Invalid target index {trg_local_idx} for {trg_partid}" # Check that the mapping is 1-to-1, that each src element is covered and maps From 8b6da6e274a109c3b2c6d39187c7e7e5b9ef95d3 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 27 Aug 2024 19:18:39 -0500 Subject: [PATCH 50/55] Fix up some docstring issues in simutil --- mirgecom/simutil.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index 15566f97e..e2a84cb74 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -1637,6 +1637,7 @@ def interdecomposition_overlap(target_decomp_map, source_decomp_map, This data structure is useful for mapping the solution data from the old decomp pkl restart files to the new decomp solution arrays. """ + src_part_to_els = invert_decomp(source_decomp_map) trg_part_to_els = invert_decomp(target_decomp_map) ipmap = interdecomposition_mapping(target_decomp_map, source_decomp_map) @@ -1664,22 +1665,22 @@ def multivolume_interdecomposition_overlap(src_decomp_map, trg_decomp_map, src_multivol_decomp_map, trg_multivol_decomp_map, return_ranks=None): """ - Construct local-to-local index mapping for overlapping deconmps. + Construct local-to-local index mapping for overlapping decomps. Parameters ---------- - src_decomp_map: dict - Source decomposition map {rank: [elements]} - trg_decomp_map: dict - Target decomposition map {rank: [elements]} - src_multivol_decomp_map: dict - Source multivolume decomposition map {PartID: np.array(elements)} - trg_multivol_decomp_map: dict - Target multivolume decomposition map {PartID: np.array(elements)} + src_decomp_map: dict + Source decomposition map {rank: [elements]} + trg_decomp_map: dict + Target decomposition map {rank: [elements]} + src_multivol_decomp_map: dict + Source multivolume decomposition map {PartID: np.array(elements)} + trg_multivol_decomp_map: dict + Target multivolume decomposition map {PartID: np.array(elements)} Returns ------- - A dictionary with structure: + A dictionary with structure: { trg_partid: { src_partid: { @@ -1688,6 +1689,7 @@ def multivolume_interdecomposition_overlap(src_decomp_map, trg_decomp_map, } } """ + # If no specific ranks are provided, consider all ranks in the target decomp if return_ranks is None: return_ranks = list(trg_decomp_map.keys()) From f4a347ab31d88455daf50c72df31fa69c68f7cca Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Tue, 27 Aug 2024 22:56:22 -0500 Subject: [PATCH 51/55] Fix up some doc warnings. --- mirgecom/simutil.py | 51 +++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/mirgecom/simutil.py b/mirgecom/simutil.py index e2a84cb74..202f30a8f 100644 --- a/mirgecom/simutil.py +++ b/mirgecom/simutil.py @@ -1626,18 +1626,23 @@ def interdecomposition_overlap(target_decomp_map, source_decomp_map, target-part-specific local indexes to the source-part-specific local index of for the corresponding element. - { targ_part_1 : { src_part_1 : { local_el_index : remote_el_index, ... }, - src_part_2 : { local_el_index : remote_el_index, ... }, - ... - }, - targ_part_2 : { ... }, - ... - } + Example dictionary structure: + + .. code-block:: python + + { + targ_part_1 : { + src_part_1 : { local_el_index : remote_el_index, ... }, + src_part_2 : { local_el_index : remote_el_index, ... }, + ... + }, + targ_part_2 : { ... }, + ... + } This data structure is useful for mapping the solution data from the old decomp pkl restart files to the new decomp solution arrays. """ - src_part_to_els = invert_decomp(source_decomp_map) trg_part_to_els = invert_decomp(target_decomp_map) ipmap = interdecomposition_mapping(target_decomp_map, source_decomp_map) @@ -1665,31 +1670,37 @@ def multivolume_interdecomposition_overlap(src_decomp_map, trg_decomp_map, src_multivol_decomp_map, trg_multivol_decomp_map, return_ranks=None): """ - Construct local-to-local index mapping for overlapping decomps. + Construct local-to-local index mapping for overlapping decompositions. Parameters ---------- src_decomp_map: dict - Source decomposition map {rank: [elements]} + Source decomposition map {rank: [elements]}. + trg_decomp_map: dict - Target decomposition map {rank: [elements]} + Target decomposition map {rank: [elements]}. + src_multivol_decomp_map: dict - Source multivolume decomposition map {PartID: np.array(elements)} + Source multivolume decomposition map {PartID: np.array(elements)}. + trg_multivol_decomp_map: dict - Target multivolume decomposition map {PartID: np.array(elements)} + Target multivolume decomposition map {PartID: np.array(elements)}. Returns ------- - A dictionary with structure: - { - trg_partid: { - src_partid: { - trg_local_el_index: src_local_el_index + dict + A dictionary with the following structure + + .. code-block:: python + + { + trg_partid: { + src_partid: { + trg_local_el_index: src_local_el_index + } } } - } """ - # If no specific ranks are provided, consider all ranks in the target decomp if return_ranks is None: return_ranks = list(trg_decomp_map.keys()) From 37213012078d8d9d0f71df24c3f8b6b8f015dda9 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 28 Aug 2024 07:32:00 -0500 Subject: [PATCH 52/55] Add some docs for m-to-n-restart --- doc/support/m-to-n-restart.rst | 68 ++++++++++++++++++++++++++++++++++ doc/support/support.rst | 1 + 2 files changed, 69 insertions(+) create mode 100644 doc/support/m-to-n-restart.rst diff --git a/doc/support/m-to-n-restart.rst b/doc/support/m-to-n-restart.rst new file mode 100644 index 000000000..5fa971a1c --- /dev/null +++ b/doc/support/m-to-n-restart.rst @@ -0,0 +1,68 @@ +M-to-N Restart Guide +===================== + +This document describes how to perform an m-to-n restart, which involves restarting a parallel application on `N` processors from restart files that were generated on `M` processors. This process requires creating configuration files that provide mappings from the `M`-partitioning of the discrete geometry to the `N`-partitioning. + +Overview +-------- + +In a parallel |mirgecom| simulation, the geometry and solution data are partitioned across multiple processors. When restarting a simulation with a different number of processors, it is necessary to re-partition the data. This guide covers the steps and tools needed to achieve this, including the use of `meshdist` and `redist` utilities. + +**Key Utilities:** + +- |mirgecom|: The simulation application generates restart files on `M` processors +- **meshdist** (`bin/meshdist.py`): Reads the `gmsh`-format mesh source file, and creates mapping files and mesh PKL files for both `M` and `N` partitions. +- **redist** (`bin/redist.py`): Reads the output of the `meshdist` runs and the |mirgecom| user's simulation application restart files generated on `M` processors, and generates a |mirgecom| restart dataset for `N` processors. + +Workflow +-------- + +The general workflow for an m-to-n restart involves the following steps: + +0. **Run `mirgecom` on `M` processors**: This step generates restart data specific for `M`-partitions/processors in `M` PKL files (one for each simulation rank). + This step creates, for example, restart files with names formatted like `--.pkl` such as the following for step number `12` for a 100-rank simulation: + + .. code-block:: bash + + -000000012-00000.pkl + ... + -000000012-00099.pkl + + +1. **Run `meshdist` for `M` partitions**: Generate the initial mapping and PKL files for the `M`-partitioned geometry. + + .. code-block:: bash + + mpiexec -n P python -m mpi4py bin/meshdist.py [-w] [-1] -d -s -c -n M -o -z + + The `-1` option will turn on 1d partitioning, and the `-z` option sets the 1d partitioning tolerance for partition imbalance. The default is `1%` (i.e. the partitions sizes (number of elements) will be within `1%` of each other). It is important that these partitioning parameters match those that were used by your |mirgecom| simulation run. This step will create mesh files with filenames formatted by `/_mesh_np_rank.pkl`, and mapping files with filenames formatted by `/_mesh_*decomp_np.pkl` + + .. warning:: + + The `meshdist` utility is specific to CEESD prediction, and would likely need customization or generalization for use in other cases. Specifically, it will automatically look for the `fluid` volume, and the `-w` option sets multivolume to `ON` and looks for the `wall`, `wall-insert`, and `wall-surround` volumes inside the mesh. + + .. note:: + + The `meshdist` utility will run on any number of processors `P`. If `P > M` (or the number of ranks specified by the `-n` option), then the additional processors will sit idle. Larger meshes can benefit from using `P > 1`, and very large meshes will require processing on larger resource sets due to platorm memory constraints. + +2. **Run `meshdist` for `N` partitions**: Generate the mapping and PKL files for the `N`-partitioned geometry. + + .. code-block:: bash + + mpiexec -n P python -m mpi4py bin/meshdist.py [-w] [-1] -d -s -c -n N -o -z + + +3. **Run `redist` to create the restart dataset for `N` processors**: This step will use the outputs from the two `meshdist` runs and the user's restart files. + + .. code-block:: bash + + mpiexec -n P python -m mpi4py bin/redist.py -m M -n N -s -t -o -i + + + The `redist` utility will read in the files created in the `M`-specific `meshdist` run, and the `N`-specific `meshdist` run, which shall be found in the `` and `` directories, respectively. It will find the existing |mirgecom| restart dataset (for `M` ranks) in the `` directory with filenames formatted as `_.pkl`. (For the example 100-rank |mirgecom| dataset above, one should specify `-i /_` to `redist`) Upon successful completion `redist` will write a new restart dataset to `/_.pkl`. + + + .. note:: + + The m-to-n restart procedure should make no changes to the |mirgecom| solution. It should only "re-partition" the existing solution to a different partitioning. There should be no transient introduced into the simulation upon restart. + diff --git a/doc/support/support.rst b/doc/support/support.rst index 4a2b0653b..b06fa40d3 100644 --- a/doc/support/support.rst +++ b/doc/support/support.rst @@ -15,3 +15,4 @@ Simulation Support math operators thermochem + m-to-n-restart From b7123761b09334e892907edc6647cb956dd9a978 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 28 Aug 2024 07:32:26 -0500 Subject: [PATCH 53/55] Fix bug in imbalance tol option --- bin/meshdist.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/meshdist.py b/bin/meshdist.py index f09bf0683..899cebc2c 100755 --- a/bin/meshdist.py +++ b/bin/meshdist.py @@ -66,10 +66,12 @@ class MyRuntimeError(RuntimeError): def main(actx_class, mesh_source=None, ndist=None, dim=None, output_path=None, log_path=None, casename=None, use_1d_part=None, use_wall=False, - imba_tol=0.01): + imba_tol=None): """The main function.""" if mesh_source is None: raise ApplicationOptionsError("Missing mesh source file.") + if imba_tol is None: + imba_tol = .01 mesh_source.strip("'") From 6084eb13692c7fde52b6afa017ad396d477a1d18 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 28 Aug 2024 09:58:57 -0500 Subject: [PATCH 54/55] Update mirgecom/logging_quantities.py per MD review Co-authored-by: Matthias Diener --- mirgecom/logging_quantities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirgecom/logging_quantities.py b/mirgecom/logging_quantities.py index 1a59999fe..09c915bad 100644 --- a/mirgecom/logging_quantities.py +++ b/mirgecom/logging_quantities.py @@ -109,7 +109,7 @@ def logmgr_add_cl_device_info(logmgr: LogManager, queue: cl.CommandQueue) -> Non logmgr.set_constant("cl_platform_version", dev.platform.version) -def logmgr_add_simulation_info(logmgr: LogManager, sim_info) -> None: +def logmgr_add_simulation_info(logmgr: LogManager, sim_info: Dict[str, Any) -> None: """Add some user-defined information to the logpyle output.""" for field_name in sim_info: logmgr.set_constant(field_name, sim_info[field_name]) From 12785f4e1dedf2a81aff0a09a9538df54f0ed685 Mon Sep 17 00:00:00 2001 From: Mike Campbell Date: Wed, 28 Aug 2024 10:04:25 -0500 Subject: [PATCH 55/55] Deflake8 --- mirgecom/logging_quantities.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mirgecom/logging_quantities.py b/mirgecom/logging_quantities.py index 09c915bad..67cf65527 100644 --- a/mirgecom/logging_quantities.py +++ b/mirgecom/logging_quantities.py @@ -52,7 +52,7 @@ from grudge.discretization import DiscretizationCollection import pyopencl as cl -from typing import Optional, Callable, Union, Tuple +from typing import Optional, Callable, Union, Tuple, Dict, Any import numpy as np from grudge.dof_desc import DD_VOLUME_ALL @@ -109,7 +109,8 @@ def logmgr_add_cl_device_info(logmgr: LogManager, queue: cl.CommandQueue) -> Non logmgr.set_constant("cl_platform_version", dev.platform.version) -def logmgr_add_simulation_info(logmgr: LogManager, sim_info: Dict[str, Any) -> None: +def logmgr_add_simulation_info(logmgr: LogManager, + sim_info: Dict[str, Any]) -> None: """Add some user-defined information to the logpyle output.""" for field_name in sim_info: logmgr.set_constant(field_name, sim_info[field_name])