Restricting Cmpi to Relevant Processes in 3D Tree Extraction

[ricardofrantz]

Hello,

I am working on extracting planes from a 3D tree, where only a subset of processes contain the relevant plane data. Unfortunately, when I attempt to save the planes to a single file, some blocks are missing.

We suspect that this issue arises because Cmpi is being called for all processes, even though the data is only available on a few. This occurs in a cylindrical case using the tranche strategy.

Is there a way to restrict Cmpi to only the processes that contain the relevant data?

Here is my routine:

def save_planes(t, planes_fldr, snap_num, rank=0, hlo_rhs=0, time_dict=None, mpi=False, axes=['z'], target=0):

    n = time_dict["n"]
    zones = Int.getZones(t)
    if "z" in axes:
        coords = getValueFromName(zones[0], "CoordinateZ")
    if "x" in axes:
        coords = getValueFromName(zones[0], "CoordinateX")
    if "y" in axes:
        coords = getValueFromName(zones[0], "CoordinateY")

    # Check if target is within the range of this processor's domain
    if coords[hlo_rhs,hlo_rhs,0] <= target <= coords[hlo_rhs,hlo_rhs,-1]:
        # Find the largest index where z <= target

        array = coords[hlo_rhs,hlo_rhs,:] #-> [0.    0.025 0.05  0.075 0.1   0.125]
        p = np.argwhere(array <= target)[-1][0] #--> [[0][1][2]]

        # Extract plane at this index for each zone
        plane = []
        for z in zones:
            if "x" in axes:  # (imin,jmin,kmin), (imax,jmax,kmax)
                plane.append(T.subzone(z, (p, 1, 1), (p, -1, -1)))
            elif "y" in axes:
                plane.append(T.subzone(z, (1, p, 1), (-1, p, -1)))
            elif "z" in axes:
                plane.append(T.subzone(z, (1, 1, p), (-1, -1, p)))

        one_file_per_time = False
        if one_file_per_time:
            file_name = f"{planes_fldr}plane_{axes[0]}_{target}_{rank}.cgns"
            base_name = f"t{n}"

            # Check if the file exists first instead of using try-except
            if os.path.isfile(file_name):
                # File exists -> t > 0
                plane_tree = C.convertFile2PyTree(file_name)
                plane_tree = C.addBase2PyTree(plane_tree, base_name)
            else:
                # File doesn't exist, create from scratch
                ensure_folder_exists(planes_fldr, rank=rank if mpi else None)
                plane_tree = C.newPyTree([base_name]) # -> t0
                C._addState(plane_tree, "EquationDimension", 2)

            # base_node = Int.getNodeFromName(t, base_name)
            # Get the base node to add planes to
            #for z in Int.getZones(plane_tree):
            Int._append(plane_tree, plane, base_name)

            # ISSUE HERE !
            #Cmpi.convertPyTree2File(plane_tree, file_name) # --> single file (missing blocks)
            #
            C.convertPyTree2File(plane_tree, file_name) # works but makes a lot of files

            print0(rank, f"Finished saving plane {axes[0]}_{target} to {file_name}")
            
        else: # a folder per time and one file per zone (not ideal)
            # lets create a folder structure
            ensure_folder_exists(planes_fldr, rank=rank if mpi else None)

            # Create a more organized folder structure
            # Format: planes_fldr/axis_target/t{n}/
            axis_folder = f"{planes_fldr}{axes[0]}_{target}/"
            ensure_folder_exists(axis_folder, rank=rank if mpi else None)

            time_folder = f"{axis_folder}t{n}/"
            ensure_folder_exists(time_folder, rank=rank if mpi else None)

            # Save the file in the organized folder structure
            file_name = f"{time_folder}plane_{rank}.cgns"
            C.convertPyTree2File(plane, file_name)

            print0(rank, f"Saved plane {axes[0]}_{target} at t{n} to {file_name}", flush=True)

[ricardofrantz]

Here is a photo of the plane using Cmpi with the missing blocks: