Skip to content

Commit

Permalink
CreateMeshes: Initialize brickwall with lazy bricks
Browse files Browse the repository at this point in the history
  • Loading branch information
stuarteberg committed Nov 2, 2023
1 parent b2cc148 commit 974c073
Showing 1 changed file with 12 additions and 2 deletions.
14 changes: 12 additions & 2 deletions flyemflows/workflow/createmeshes.py
Original file line number Diff line number Diff line change
Expand Up @@ -611,15 +611,25 @@ def init_bricks_ddf(self, volume_service, subset_labels):
sbm = None

with Timer(msg, logger):
# By defaut, aim for 1 GB RDD partitions when loading segmentation (see default above)
# By default, aim for 1 GB RDD partitions when loading segmentation (see default above)
psize = self.config["createmeshes"]["partition-volume-size"]
target_partition_size_voxels = psize // np.uint64().nbytes

# Apply halo WHILE downloading the data.
# TODO: Allow the user to configure whether or not the halo should
# be fetched from the outset, or added after the blocks are loaded.
halo = self.config["createmeshes"]["halo"]
brickwall = BrickWall.from_volume_service(volume_service, 0, None, self.client, target_partition_size_voxels, halo, sbm, compression='lz4_2x')
brickwall = BrickWall.from_volume_service(
volume_service,
0,
None,
self.client,
target_partition_size_voxels,
halo,
sbm,
lazy=True,
compression='lz4_2x'
)

# Convert to dask.DataFrame, including logical scan-order position as the DataFrame index.
with Timer("Loading bricks into dask DataFrame", logger):
Expand Down

0 comments on commit 974c073

Please sign in to comment.