ValueError is thrown if feather run with one cpu, but runs fine if run with more

[dmehring]

On branch 2-feather, feather will fail with a ValueError exception if client is configured to use one cpy, but will succeed if multiple cpu's are used. Images used may be found on the cv lustre system in /lustre/cv/users/dmehring/feather_bug/.
The information found below can also be found at https://docs.google.com/spreadsheets/d/1ZvdzT8XR9jrflVzXV_na_PmF1Oyt1V43Z0JEXUgrfKs/edit?gid=506616528#gid=506616528 (note that the cores parameter in the local_chan() call will have to be changed to one from the value in the spreadsheet).

Command sequence to reproduce error

from graphviper.dask.client import local_client
log_params = {'log_level':'DEBUG'}
worker_log_params = {'log_level':'DEBUG'}
viper_client = local_client(cores=1, memory_limit=""8GiB"",log_params=log_params,worker_log_params=worker_log_params)
#viper_client = local_client(serial_execution=True)
viper_client

from astroviper.imaging import feather
fds = feather(out_im={""name"": ""feathered_image.img.zarr"", ""format"": ""zarr"", ""overwrite"": True}, high_res= 'int.im', low_res= 'sd.im', sdfactor=1)

relevant software versions are

ASTROVIPER VERSION | BRANCH 2-feather, COMMIT 264dc22

GRAPHVIPER VERSION | 0.0.26
 
XRADIO VERSION | 0.0.34

The emitted stacktrace is

"* ValueError                                Traceback (most recent call last)
Cell In[2], line 6
      3 b = time.time()
      4 # input CASA images
      5 # time to run feather: 440 s
----> 6 fds = feather(out_im={""name"": ""feathered_image.img.zarr"", ""format"": ""zarr"", ""overwrite"": True}, high_res= 'int.im', low_res= 'sd.im', sdfactor=1)
      7 # input zarr images
      8 # time to run feather, 260 s
      9 # feather(outim={""name"": ""feathered_image.img.zarr"", ""format"": ""zarr"", ""overwrite"": True}, highres= 'int.zarr', lowres= 'sd.zarr', sdfactor=1)
     10 print(f""Time to run astroviper feather {time.time() - b} seconds"")

File ~/projects/astroviper/astroviper/src/astroviper/imaging/feather.py:553, in feather(out_im, high_res, low_res, sdfactor, selection, memory_per_thread, thread_info, compressor)
    551 #dask.visualize(graph, filename=""map_graph"")
    552 t0 = time.time()
--> 553 res = dask.compute(dask_graph)
    554 logger.info(""Time to compute() feather "" + str(time.time() - t0) + ""s"")
    556 import zarr, shutil

File ~/projects/astroviper/xenv/lib/python3.10/site-packages/dask/base.py:665, in compute(traverse, optimize_graph, scheduler, get, *args, **kwargs)
    662     postcomputes.append(x.__dask_postcompute__())
    664 with shorten_traceback():
--> 665     results = schedule(dsk, keys, **kwargs)
    667 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])

File ~/projects/astroviper/astroviper/src/astroviper/imaging/feather.py:259, in _feather()
    257 if input_params[""to_disk""]:
    258     for data_variable, meta in input_params[""zarr_meta""].items():
--> 259         write_chunk(featherd_img_chunk_xds,meta,parallel_dims_chunk_id,input_params[""compressor""],input_params[""image_file""])
    261     results_dict = {}
    262     return results_dict

File ~/projects/astroviper/xenv/lib/python3.10/site-packages/xradio/image/_util/_zarr/zarr_low_level.py:342, in write_chunk()
    339 else:
    340     array = img_xds[data_varaible_name].values
--> 342 write_binary_blob_to_disk(
    343     array,
    344     file_path=os.path.join(image_file, data_varaible_name, chunk_name),
    345     compressor=compressor,
    346 )

File ~/projects/astroviper/xenv/lib/python3.10/site-packages/xradio/image/_util/_zarr/zarr_low_level.py:116, in write_binary_blob_to_disk()
    114 # Encode the NumPy array using the codec
    115 logger.debug(""1. Before compressor "" + file_path)
--> 116 compressed_arr = compressor.encode(np.ascontiguousarray(arr))
    118 logger.debug(""2. Before makedir"")
    119 # Ensure the directory exists before saving the file

File numcodecs/blosc.pyx:559, in numcodecs.blosc.Blosc.encode()

File ~/projects/astroviper/xenv/lib/python3.10/site-packages/numcodecs/compat.py:155, in ensure_contiguous_ndarray()
    126 def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True) -> np.array:
    127     """"""Convenience function to coerce `buf` to a numpy array, if it is not already a
    128     numpy array. Also ensures that the returned value exports fully contiguous memory,
    129     and supports the new-style buffer interface. If the optional max_buffer_size is
   (...)
    151     return a view on memory exported by `buf`.
    152     """"""
    154     return ensure_ndarray(
--> 155         ensure_contiguous_ndarray_like(
    156             buf, max_buffer_size=max_buffer_size, flatten=flatten
    157         )
    158     )

File ~/projects/astroviper/xenv/lib/python3.10/site-packages/numcodecs/compat.py:121, in ensure_contiguous_ndarray_like()
    119 if max_buffer_size is not None and arr.nbytes > max_buffer_size:
    120     msg = ""Codec does not support buffers of > {} bytes"".format(max_buffer_size)
--> 121     raise ValueError(msg)
    123 return arr

ValueError: Codec does not support buffers of > 2147483647 bytes

In [3]: Successful readonly open of default-locked table sd.im: 1 columns, 1 rows
[2024-08-25 15:07:02,550]     INFO    worker_0:  J2000 found as native reference frame in CASA image This corresponds to FK5(equinox=""J2000"") in astropy. Metadata will be written appropriately "