Skip to content

Commit

Permalink
rest_vol_dataspace: (fix) Memory-side hyperslab
Browse files Browse the repository at this point in the history 
- Various improvements of the function RV_dataspace_is_contiguous()
  function.
- Implemented a test in which the memory-side hyperslab is
  non-contiguous in memory.
  • Loading branch information
@jwsblokland
jwsblokland committed Sep 1, 2023
1 parent a3974cb commit 8ff1485
Show file tree
Hide file tree
Showing 2 changed files with 188 additions and 56 deletions.
111 changes: 58 additions & 53 deletions src/rest_vol_dataset.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ static herr_t RV_convert_dataspace_selection_to_string(hid_t space_id, char **se
size_t *selection_string_len, hbool_t req_param);

/* Helper function for dataspace selection */
static hbool_t RV_dataspace_selection_is_contiguous(hid_t space_id);
static htri_t RV_dataspace_selection_is_contiguous(hid_t space_id);

/* Conversion function to convert one or more rest_obj_ref_t objects into a binary buffer for data transfer */
static herr_t RV_convert_obj_refs_to_buffer(const rv_obj_ref_t *ref_array, size_t ref_array_len,
Expand Down Expand Up @@ -715,6 +715,7 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t mem_spac
curl_off_t write_len;
hssize_t mem_select_npoints, file_select_npoints, offset;
hbool_t is_transfer_binary = FALSE;
htri_t contiguous = FALSE;
htri_t is_variable_str;
size_t host_header_len = 0;
size_t write_body_len = 0;
Expand Down Expand Up @@ -833,7 +834,10 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t mem_spac
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "memory datatype is invalid");

write_body_len = (size_t)file_select_npoints * dtype_size;
if (!RV_dataspace_selection_is_contiguous(mem_space_id[0])) {
if ((contiguous = RV_dataspace_selection_is_contiguous(mem_space_id[0])) < 0)
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL,
"Unable to determine if the dataspace selection is contiguous");
if (!contiguous) {
if (NULL == (write_body = (char *)RV_malloc(write_body_len)))
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL,
"can't allocate space for the 'write_body' values");
Expand Down Expand Up @@ -3770,16 +3774,17 @@ dataset_read_scatter_op(const void **src_buf, size_t *src_buf_bytes_used, void *
*
* Purpose: Checks if the specified dataspace in a contiguous selection.
*
* Return: TRUE if the selection is contiguous otherwise FALSE.
* Return: TRUE or FALSE if the selection is contiguous or
* non-contiguous and FAIL if it is unable to determine it.
*
* Programmer: Jan-Willem Blokland
* August, 2023
*/
static hbool_t
static htri_t
RV_dataspace_selection_is_contiguous(hid_t space_id)
{
hbool_t ret_value = true;
hbool_t whole = true;
htri_t ret_value = TRUE;
hbool_t whole = TRUE;
hsize_t *dims = NULL;
hsize_t *start = NULL;
hsize_t *stride = NULL;
Expand All @@ -3792,64 +3797,64 @@ RV_dataspace_selection_is_contiguous(hid_t space_id)
if ((npoints = H5Sget_select_npoints(space_id)) < 0)
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve number of selected points");
if (npoints < 2)
FUNC_GOTO_DONE(true);
FUNC_GOTO_DONE(TRUE);

if ((ndims = H5Sget_simple_extent_ndims(space_id)) < 0)
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve dataspace dimensionality");
if (!ndims)
FUNC_GOTO_DONE(true);
FUNC_GOTO_DONE(TRUE);

switch (H5Sget_select_type(space_id)) {
case H5S_SEL_HYPERSLABS:
if (NULL == (dims = (hsize_t *)RV_malloc((size_t)ndims * sizeof(*dims))))
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL,
"can't allocate space for dimension 'dims' values");

if (H5Sget_simple_extent_dims(space_id, dims, NULL) < 0)
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace dimension size");

if (NULL == (start = (hsize_t *)RV_malloc((size_t)ndims * sizeof(*start))))
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL,
"can't allocate space for hyperslab selection 'start' values");
if (NULL == (stride = (hsize_t *)RV_malloc((size_t)ndims * sizeof(*stride))))
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL,
"can't allocate space for hyperslab selection 'stride' values");
if (NULL == (count = (hsize_t *)RV_malloc((size_t)ndims * sizeof(*count))))
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL,
"can't allocate space for hyperslab selection 'count' values");
if (NULL == (block = (hsize_t *)RV_malloc((size_t)ndims * sizeof(*block))))
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL,
"can't allocate space for hyperslab selection 'block' values");
if (H5S_SEL_HYPERSLABS == H5Sget_select_type(space_id)) {
if (NULL == (dims = (hsize_t *)RV_malloc((size_t)ndims * sizeof(*dims))))
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL,
"can't allocate space for dimension 'dims' values");

if (nblocks = H5Sget_select_hyper_nblocks(space_id) < 0)
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get number of hyperslab blocks");
if (H5Sget_simple_extent_dims(space_id, dims, NULL) < 0)
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace dimension size");

if (H5Sget_regular_hyperslab(space_id, start, stride, count, block) < 0)
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get regular hyperslab selection");
if (NULL == (start = (hsize_t *)RV_malloc((size_t)ndims * sizeof(*start))))
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL,
"can't allocate space for hyperslab selection 'start' values");
if (NULL == (stride = (hsize_t *)RV_malloc((size_t)ndims * sizeof(*stride))))
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL,
"can't allocate space for hyperslab selection 'stride' values");
if (NULL == (count = (hsize_t *)RV_malloc((size_t)ndims * sizeof(*count))))
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL,
"can't allocate space for hyperslab selection 'count' values");
if (NULL == (block = (hsize_t *)RV_malloc((size_t)ndims * sizeof(*block))))
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL,
"can't allocate space for hyperslab selection 'block' values");

if (nblocks = H5Sget_select_hyper_nblocks(space_id) < 0)
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get number of hyperslab blocks");

if (H5Sget_regular_hyperslab(space_id, start, stride, count, block) < 0)
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get regular hyperslab selection");

/* For contiguous, the stride should be 1. */
for (i = 0; i < ndims; i++) {
if (stride[i] > 1)
FUNC_GOTO_DONE(FALSE);
}

/* For contiguous, the stride should be 1. */
for (i = 0; i < ndims; i++) {
if (stride[i] > 1)
FUNC_GOTO_DONE(false);
if (nblocks > 1) {
/* Multiple blocks: count should be 1 except for the last dimension (fastest) */
for (i = 0; i < ndims - 1; i++) {
if (count[i] > 1)
FUNC_GOTO_DONE(FALSE);
}
}

if (nblocks > 1) {
/* Multiple blocks: count should be 1 except for the last dimension (fastest) */
for (i = 0; i < ndims - 1; i++) {
if (count[i] > 1)
FUNC_GOTO_DONE(false);
}
}
/* For contiguous, all faster running dimensions than the current dimension should be selected
* completely */
whole = (start[ndims - 1] == 0) && (count[ndims - 1] * block[ndims - 1] == dims[ndims - 1]);
for (i = ndims - 2; i >= 0; i--) {
if ((dims[i] > 1) && (count[i] * block[i] > 1) && !whole)
FUNC_GOTO_DONE(FALSE);

/* For contiguous, all faster running dimensions than the current dimension should be selected
* completely */
whole = (start[ndims - 1] == 0) && (count[ndims - 1] * block[ndims - 1] == dims[ndims - 1]);
for (i = ndims - 2; i > 0; i--) {
whole = whole && (start[i] == 0) && (count[i] * block[i] == dims[i]);
if ((dims[i - 1] > 1) && (count[i - 1] * block[i - 1] > 1) && !whole)
FUNC_GOTO_DONE(false);
}
} /* end switch */
whole = whole && (start[i] == 0) && (count[i] * block[i] == dims[i]);
}
} /* end if */

done:
if (block)
Expand Down
133 changes: 130 additions & 3 deletions test/test_rest_vol.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8546,7 +8546,7 @@ test_write_dataset_data_verification(void)
{
hssize_t space_npoints;
hsize_t dims[DATASET_DATA_VERIFY_WRITE_TEST_DSET_SPACE_RANK] = {10, 10, 10};
hsize_t mdims[1];
hsize_t mdims[2];
hsize_t start[DATASET_DATA_VERIFY_WRITE_TEST_DSET_SPACE_RANK];
hsize_t stride[DATASET_DATA_VERIFY_WRITE_TEST_DSET_SPACE_RANK];
hsize_t count[DATASET_DATA_VERIFY_WRITE_TEST_DSET_SPACE_RANK];
Expand Down Expand Up @@ -8673,7 +8673,7 @@ test_write_dataset_data_verification(void)
}

#ifdef RV_CONNECTOR_DEBUG
puts("Writing to dataset using hyperslab selection\n");
puts("Writing to dataset using hyperslab selection - contiguous\n");
#endif

data_size = dims[1] * 2 * DATASET_DATA_VERIFY_WRITE_TEST_DSET_DTYPESIZE;
Expand Down Expand Up @@ -8775,7 +8775,134 @@ test_write_dataset_data_verification(void)

if (memcmp(data, read_buf, data_size)) {
H5_FAILED();
printf(" hyperslab selection data verification failed\n");
printf(" hyperslab selection data (contiguous) verification failed\n");
goto error;
}

if (data) {
free(data);
data = NULL;
}

if (write_buf) {
free(write_buf);
write_buf = NULL;
}

if (read_buf) {
free(read_buf);
read_buf = NULL;
}

#ifdef RV_CONNECTOR_DEBUG
puts("Writing to dataset using hyperslab selection - non-contiguous\n");
#endif

data_size = dims[1] * 2 * DATASET_DATA_VERIFY_WRITE_TEST_DSET_DTYPESIZE;

if (NULL == (write_buf = malloc(data_size)))
TEST_ERROR

for (i = 0; i < data_size / DATASET_DATA_VERIFY_WRITE_TEST_DSET_DTYPESIZE; i = i + 2)
((int *)write_buf)[i] = 67;
for (i = 1; i < data_size / DATASET_DATA_VERIFY_WRITE_TEST_DSET_DTYPESIZE; i = i + 2)
((int *)write_buf)[i] = 68;

for (i = 0, data_size = 1; i < DATASET_DATA_VERIFY_WRITE_TEST_DSET_SPACE_RANK; i++)
data_size *= dims[i];
data_size *= DATASET_DATA_VERIFY_WRITE_TEST_DSET_DTYPESIZE;

if (NULL == (data = malloc(data_size)))
TEST_ERROR

if (H5Dread(dset_id, DATASET_DATA_VERIFY_WRITE_TEST_DSET_DTYPE, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) <
0) {
H5_FAILED();
printf(" couldn't read from dataset\n");
goto error;
}

for (i = 2; i < 3; i++) {
size_t j;

for (j = 0; j < dims[1]; j++)
((int *)data)[(i * dims[1] * dims[2]) + (j * dims[2])] = 67;
}

/* Write to third row of dataset */
mdims[0] = dims[1];
mdims[1] = 2;
start[0] = start[1] = 0;
stride[0] = stride[1] = 1;
count[0] = dims[1];
count[1] = 1;
block[0] = block[1] = 1;
if ((mspace_id = H5Screate_simple(2, mdims, NULL)) < 0)
TEST_ERROR
if (H5Sselect_hyperslab(mspace_id, H5S_SELECT_SET, start, stride, count, block) < 0)
TEST_ERROR

start[0] = 2;
start[1] = start[2] = 0;
stride[0] = stride[1] = stride[2] = 1;
count[0] = 1;
count[1] = dims[1];
count[2] = 1;
block[0] = block[1] = block[2] = 1;
if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0)
TEST_ERROR

if (H5Dwrite(dset_id, DATASET_DATA_VERIFY_WRITE_TEST_DSET_DTYPE, mspace_id, fspace_id, H5P_DEFAULT,
write_buf) < 0) {
H5_FAILED();
printf(" couldn't write to dataset\n");
goto error;
}

if (H5Sclose(mspace_id) < 0)
TEST_ERROR
if (H5Sclose(fspace_id) < 0)
TEST_ERROR
if (H5Dclose(dset_id) < 0)
TEST_ERROR

if ((dset_id = H5Dopen2(file_id, "/" DATASET_TEST_GROUP_NAME "/" DATASET_DATA_VERIFY_WRITE_TEST_DSET_NAME,
H5P_DEFAULT)) < 0) {
H5_FAILED();
printf(" couldn't open dataset\n");
goto error;
}

if ((fspace_id = H5Dget_space(dset_id)) < 0) {
H5_FAILED();
printf(" couldn't get dataset dataspace\n");
goto error;
}

if ((space_npoints = H5Sget_simple_extent_npoints(fspace_id)) < 0) {
H5_FAILED();
printf(" couldn't get dataspace num points\n");
goto error;
}

if (NULL == (read_buf = malloc((hsize_t)space_npoints * DATASET_DATA_VERIFY_WRITE_TEST_DSET_DTYPESIZE)))
TEST_ERROR

#ifdef RV_CONNECTOR_DEBUG
puts("Verifying that the data that comes back is correct after writing to the dataset using a hyperslab "
"selection\n");
#endif

if (H5Dread(dset_id, DATASET_DATA_VERIFY_WRITE_TEST_DSET_DTYPE, H5S_ALL, H5S_ALL, H5P_DEFAULT, read_buf) <
0) {
H5_FAILED();
printf(" couldn't read from dataset\n");
goto error;
}

if (memcmp(data, read_buf, data_size)) {
H5_FAILED();
printf(" hyperslab selection data (non-contiguous) verification failed\n");
goto error;
}

Expand Down

0 comments on commit 8ff1485

Please sign in to comment.