diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dc60dacf..032250f5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -119,7 +119,12 @@ jobs: sudo apt update sudo apt install valgrind working-directory: ${{ github.workspace }} - + + # Requests 2.32.0 breaks requests-unixsocket, used by HSDS for socket connections + - name: Fix requests version + run: | + pip install requests==2.31.0 + - name: Start HSDS if: ${{ matrix.endpoint != 'http://127.0.0.1:5101'}} run: | @@ -133,7 +138,7 @@ jobs: ROOT_DIR=${{github.workspace}}/hsdadata ./runall.sh --no-docker 1 & sleep 10 working-directory: ${{github.workspace}}/hsds - + - name: Test HSDS if: ${{matrix.endpoint != 'http://127.0.0.1:5101'}} run: | @@ -240,7 +245,12 @@ jobs: sudo apt update sudo apt install valgrind working-directory: ${{ github.workspace }} - + + # Requests 2.32.0 breaks requests-unixsocket, used by HSDS for socket connections + - name: Fix requests version + run: | + pip install requests==2.31.0 + - name: Start HSDS if: ${{ matrix.endpoint != 'http://127.0.0.1:5101'}} run: | @@ -254,7 +264,7 @@ jobs: ROOT_DIR=${{github.workspace}}/hsdadata ./runall.sh --no-docker 1 & sleep 10 working-directory: ${{github.workspace}}/hsds - + - name: Test HSDS if: ${{matrix.endpoint != 'http://127.0.0.1:5101'}} run: | diff --git a/src/rest_vol.c b/src/rest_vol.c index ceb3a57f..dba6f201 100644 --- a/src/rest_vol.c +++ b/src/rest_vol.c @@ -3656,6 +3656,7 @@ RV_curl_multi_perform(CURL *curl_multi_handle, dataset_transfer_info *transfer_i int maxfd = -1; long timeout_ms = 0; struct timeval timeout; + hid_t vlen_buf_space = H5I_INVALID_HID; if ((failed_handles_to_retry = calloc(count, sizeof(CURL *))) == NULL) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, @@ -3757,6 +3758,8 @@ RV_curl_multi_perform(CURL *curl_multi_handle, dataset_transfer_info *transfer_i fail_count++; } else if (response_code == 200) { + H5T_class_t dtype_class = H5T_NO_CLASS; + num_finished++; succeed_count++; @@ -3794,11 +3797,71 @@ RV_curl_multi_perform(CURL *curl_multi_handle, dataset_transfer_info *transfer_i transfer_info[handle_index].curl_easy_handle = NULL; if (transfer_info[handle_index].transfer_type == WRITE) { - RV_free(transfer_info[handle_index].u.write_info.write_body); - transfer_info[handle_index].u.write_info.write_body = NULL; + if (transfer_info[handle_index].tconv_buf) { + htri_t has_vlen = FALSE; + + if ((has_vlen = + H5Tdetect_class(transfer_info[handle_index].mem_type_id, H5T_VLEN)) < 0) + FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "can't check if dtype contains vlen"); + + /* Clean up memory allocated by type conversion of vlen types */ + if (has_vlen > 0) { + /* Buffer was gathered before type conversion, so we can manually free vlen + * memory by iteration */ + hssize_t num_elems = 0; + if ((num_elems = H5Sget_select_npoints( + transfer_info[handle_index].mem_space_id)) <= 0) + FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "can't get number of elements in dataspace"); + + /* Vlen buffer is packed, so generate a 1D dataspace to describe its layout */ + if ((vlen_buf_space = H5Screate_simple(1, &num_elems, NULL)) < 0) + FUNC_DONE_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL, + "can't create dataspace for vlen buffer"); + + if ((H5Treclaim(transfer_info[handle_index].mem_type_id, vlen_buf_space, + H5P_DEFAULT, transfer_info[handle_index].tconv_buf)) < 0) + FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, + "can't free vlen data from buffer"); + + if (H5Sclose(vlen_buf_space) < 0) + FUNC_DONE_ERROR(H5E_DATASPACE, H5E_CANTCLOSEOBJ, FAIL, + "can't close dataspace for vlen buffer"); + + vlen_buf_space = H5I_INVALID_HID; + } + } + + if (transfer_info[handle_index].u.write_info.gather_buf) { + RV_free(transfer_info[handle_index].u.write_info.gather_buf); + transfer_info[handle_index].u.write_info.gather_buf = NULL; + } + + if (transfer_info[handle_index].u.write_info.serialize_buf) { + RV_free(transfer_info[handle_index].u.write_info.serialize_buf); + transfer_info[handle_index].u.write_info.serialize_buf = NULL; + } + + if (transfer_info[handle_index].u.write_info.base64_encoded_values) { + RV_free(transfer_info[handle_index].u.write_info.base64_encoded_values); + transfer_info[handle_index].u.write_info.base64_encoded_values = NULL; + } + + if (transfer_info[handle_index].u.write_info.point_sel_buf) { + RV_free(transfer_info[handle_index].u.write_info.point_sel_buf); + transfer_info[handle_index].u.write_info.point_sel_buf = NULL; + } + } - RV_free(transfer_info[handle_index].u.write_info.base64_encoded_values); - transfer_info[handle_index].u.write_info.base64_encoded_values = NULL; + if (transfer_info[handle_index].tconv_buf) { + RV_free(transfer_info[handle_index].tconv_buf); + transfer_info[handle_index].tconv_buf = NULL; + } + + if (transfer_info[handle_index].bkg_buf) { + RV_free(transfer_info[handle_index].bkg_buf); + transfer_info[handle_index].bkg_buf = NULL; } RV_free(transfer_info[handle_index].request_url); @@ -3838,6 +3901,10 @@ RV_curl_multi_perform(CURL *curl_multi_handle, dataset_transfer_info *transfer_i done: RV_free(failed_handles_to_retry); + if (vlen_buf_space != H5I_INVALID_HID) + if (H5Sclose(vlen_buf_space) < 0) + FUNC_DONE_ERROR(H5E_DATASPACE, H5E_CANTCLOSEOBJ, FAIL, "can't close dataspace for vlen buffer"); + return ret_value; } diff --git a/src/rest_vol.h b/src/rest_vol.h index e4d5cc2e..ceb482c7 100644 --- a/src/rest_vol.h +++ b/src/rest_vol.h @@ -586,10 +586,14 @@ struct RV_object_t { /* Structures to hold information for cURL requests to read/write to datasets */ typedef struct dataset_write_info { - char *write_body; - char *base64_encoded_values; - curl_off_t write_len; + /* Dynamically allocated buffers for each step of the write pipeline */ + void *gather_buf; + void *base64_encoded_values; + void *serialize_buf; + void *point_sel_buf; + void *vlen_buf; upload_info uinfo; + /* Pointer to user-provided write buffer */ const void *buf; /* If writing using compound subsetting, this is a packed version of the diff --git a/src/rest_vol_dataset.c b/src/rest_vol_dataset.c index bf71d518..b5fc7cbc 100644 --- a/src/rest_vol_dataset.c +++ b/src/rest_vol_dataset.c @@ -18,15 +18,16 @@ #include "rest_vol_dataset.h" #include +/* Helpers to serialize vlen data for writes. */ +static herr_t RV_pack_vlen_data(const hvl_t *in, size_t nelems, hid_t dtype_id, void **out, size_t *out_size); +static herr_t RV_convert_point_selection_to_binary(hid_t space_id, void **out_buf, size_t *out_buf_len); + /* Set of callbacks for RV_parse_response() */ static herr_t RV_parse_dataset_creation_properties_callback(char *HTTP_response, const void *callback_data_in, void *callback_data_out); -static herr_t RV_json_values_to_binary_callback(char *HTTP_response, const void *callback_data_in, - void *callback_data_out); - -/* Internal helper for RV_json_values_to_binary_callback */ -herr_t RV_json_values_to_binary_recursive(yajl_val value_entry, hid_t dtype_id, void *value_buffer); +/* Convert a binary buffer of packed vlen data to a buffer of hvl_t */ +static herr_t RV_unpack_vlen_data(char *in, hid_t vlen_dtype_id, size_t nelems, void **out); /* Helper functions for creating a Dataset */ static herr_t RV_setup_dataset_create_request_body(void *parent_obj, const char *name, hid_t type_id, @@ -85,6 +86,7 @@ const char *value_keys[] = {"value", (const char *)0}; #define DATASET_CREATE_MAX_COMPACT_ATTRIBUTES_DEFAULT 8 #define DATASET_CREATE_MIN_DENSE_ATTRIBUTES_DEFAULT 6 #define OBJECT_REF_STRING_LEN 48 +#define DATASET_VLEN_JSON_BODY_DEFAULT_SIZE 512 /* Defines for multi-CURL related settings */ #define NUM_MAX_HOST_CONNS 10 @@ -452,6 +454,7 @@ RV_dataset_read(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spac herr_t ret_value = SUCCEED; CURL *curl_multi_handle = NULL; dataset_transfer_info *transfer_info = NULL; + H5S_sel_type sel_type = H5S_SEL_ERROR; if ((transfer_info = RV_calloc(count * sizeof(dataset_transfer_info))) == NULL) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate space for dataset transfer info"); @@ -497,7 +500,6 @@ RV_dataset_read(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spac transfer_info[i].file_type_id = ((RV_object_t *)dset[i])->u.dataset.dtype_id; transfer_info[i].resp_buffer.buffer_size = CURL_RESPONSE_BUFFER_DEFAULT_SIZE; transfer_info[i].resp_buffer.curr_buf_ptr = transfer_info[i].resp_buffer.buffer; - transfer_info[i].tconv_buf = NULL; transfer_info[i].bkg_buf = NULL; } @@ -533,7 +535,7 @@ RV_dataset_read(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spac * All or Hyperslab selection. Point selections are dealt with by POSTing the * point list as JSON in the request body. */ - is_transfer_binary = (H5T_VLEN != dtype_class) && !is_variable_str; + is_transfer_binary = !is_variable_str; /* Follow the semantics for the use of H5S_ALL */ if (H5S_ALL == transfer_info[i].mem_space_id && H5S_ALL == transfer_info[i].file_space_id) { @@ -571,22 +573,30 @@ RV_dataset_read(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spac } /* end if */ /* Since the selection in the dataset's file dataspace is not set - * to "all", convert the selection into JSON */ - - /* Retrieve the selection type to choose how to format the dataspace selection */ - if (H5S_SEL_ERROR == - (transfer_info[i].u.read_info.sel_type = H5Sget_select_type(transfer_info[i].file_space_id))) + * to "all", convert the selection into JSON or binary */ + if ((sel_type = H5Sget_select_type(transfer_info[i].file_space_id)) < 0) FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace selection type"); - is_transfer_binary = - is_transfer_binary && (H5S_SEL_POINTS != transfer_info[i].u.read_info.sel_type); - if (RV_convert_dataspace_selection_to_string(transfer_info[i].file_space_id, - &(transfer_info[i].selection_body), - &selection_body_len, is_transfer_binary) < 0) - FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTCONVERT, FAIL, - "can't convert dataspace selection to string representation"); + if (H5S_SEL_POINTS == sel_type) { + if (RV_convert_point_selection_to_binary(transfer_info[i].file_space_id, + (void **)&(transfer_info[i].selection_body), + &selection_body_len) < 0) + FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTCONVERT, FAIL, + "can't convert point selection to binary form"); + } + else { + if (RV_convert_dataspace_selection_to_string(transfer_info[i].file_space_id, + &(transfer_info[i].selection_body), + &selection_body_len, is_transfer_binary) < 0) + FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTCONVERT, FAIL, + "can't convert dataspace selection to string representation"); + } } /* end else */ + if (H5S_SEL_ERROR == + (transfer_info[i].u.read_info.sel_type = H5Sget_select_type(transfer_info[i].file_space_id))) + FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace selection type"); + /* Verify that the number of selected points matches */ if ((mem_select_npoints = H5Sget_select_npoints(transfer_info[i].mem_space_id)) < 0) FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "memory dataspace is invalid"); @@ -650,37 +660,16 @@ RV_dataset_read(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spac * added as a request parameter to the GET URL. */ if (H5S_SEL_POINTS == transfer_info[i].u.read_info.sel_type) { - /* As the dataspace-selection-to-string function is not designed to include the enclosing '{' and - * '}', since returning just the selection string to the user makes more sense if they are - * including more elements in their JSON, we have to wrap the selection body here before sending - * it off to cURL - */ - - /* Ensure we have enough space to add the enclosing '{' and '}' */ - if (NULL == (transfer_info[i].selection_body = - (char *)RV_realloc(transfer_info[i].selection_body, selection_body_len + 3))) - FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, - "can't reallocate space for point selection body"); - - /* Shift the whole string down by a byte */ - memmove(transfer_info[i].selection_body + 1, transfer_info[i].selection_body, - selection_body_len + 1); - - /* Add in the braces */ - transfer_info[i].selection_body[0] = '{'; - transfer_info[i].selection_body[selection_body_len + 1] = '}'; - transfer_info[i].selection_body[selection_body_len + 2] = '\0'; - /* Check to make sure that the size of the selection HTTP body can safely be cast to a curl_off_t */ if (sizeof(curl_off_t) < sizeof(size_t)) - ASSIGN_TO_SMALLER_SIZE(transfer_info[i].u.read_info.post_len, curl_off_t, - selection_body_len + 2, size_t) + ASSIGN_TO_SMALLER_SIZE(transfer_info[i].u.read_info.post_len, curl_off_t, selection_body_len, + size_t) else if (sizeof(curl_off_t) > sizeof(size_t)) - transfer_info[i].u.read_info.post_len = (curl_off_t)(selection_body_len + 2); + transfer_info[i].u.read_info.post_len = (curl_off_t)(selection_body_len); else ASSIGN_TO_SAME_SIZE_UNSIGNED_TO_SIGNED(transfer_info[i].u.read_info.post_len, curl_off_t, - selection_body_len + 2, size_t) + selection_body_len, size_t) if (CURLE_OK != curl_easy_setopt(transfer_info[i].curl_easy_handle, CURLOPT_POST, 1)) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, @@ -698,8 +687,14 @@ RV_dataset_read(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spac FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set cURL POST data size: %s", transfer_info[i].curl_err_buf); - transfer_info[i].curl_headers = - curl_slist_append(transfer_info[i].curl_headers, "Content-Type: application/json"); + if (!is_transfer_binary) { + transfer_info[i].curl_headers = + curl_slist_append(transfer_info[i].curl_headers, "Content-Type: application/json"); + } + else { + transfer_info[i].curl_headers = curl_slist_append(transfer_info[i].curl_headers, + "Content-Type: application/octet-stream"); + } #ifdef RV_CONNECTOR_DEBUG printf("-> Setup cURL to POST point list for dataset read\n\n"); @@ -795,20 +790,22 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa hssize_t offset = 0; size_t host_header_len = 0; size_t write_body_len = 0; + size_t mem_data_size = 0; size_t selection_body_len = 0; + size_t dest_dtype_size = 0; char *selection_body = NULL; int url_len = 0; herr_t ret_value = SUCCEED; dataset_transfer_info *transfer_info = NULL; CURL *curl_multi_handle = NULL; + curl_off_t write_len = 0; - hbool_t needs_tconv = FALSE; - size_t file_type_size = 0; - size_t mem_type_size = 0; - hbool_t fill_bkg = FALSE; - const void *buf_to_write = NULL; + hbool_t needs_tconv = FALSE; + size_t file_type_size = 0; + size_t mem_type_size = 0; + hbool_t fill_bkg = FALSE; - hbool_t has_selection_body = FALSE; + hbool_t has_selection_in_url = FALSE; char cmpd_query[URL_MAX_LENGTH]; char *member_name = NULL; char *url_encoded_member_name = NULL; @@ -821,7 +818,6 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa /* Initialize arrays */ for (size_t i = 0; i < count; i++) { - if (!buf[i]) FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "a given write buffer was NULL"); @@ -858,11 +854,8 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set up non global curl write data: %s", transfer_info[i].curl_err_buf); - transfer_info[i].u.write_info.write_body = NULL; - transfer_info[i].u.write_info.base64_encoded_values = NULL; - transfer_info[i].dataset = (RV_object_t *)dset[i]; - transfer_info[i].u.write_info.buf = buf[i]; - transfer_info[i].transfer_type = WRITE; + transfer_info[i].dataset = (RV_object_t *)dset[i]; + transfer_info[i].transfer_type = WRITE; transfer_info[i].mem_space_id = _mem_space_id[i]; transfer_info[i].file_space_id = _file_space_id[i]; @@ -876,6 +869,14 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa transfer_info[i].bkg_buf = NULL; transfer_info[i].u.write_info.dense_cmpd_subset_dtype_id = H5I_INVALID_HID; + transfer_info[i].u.write_info.gather_buf = NULL; + transfer_info[i].u.write_info.serialize_buf = NULL; + transfer_info[i].u.write_info.vlen_buf = NULL; + transfer_info[i].u.write_info.base64_encoded_values = NULL; + transfer_info[i].u.write_info.point_sel_buf = NULL; + transfer_info[i].u.write_info.uinfo.buffer = (const void *)buf[i]; + transfer_info[i].u.write_info.uinfo.buffer_size = 0; + transfer_info[i].u.write_info.uinfo.bytes_sent = 0; } #ifdef RV_CONNECTOR_DEBUG @@ -899,19 +900,18 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa /* Iterate over datasets to write to */ for (size_t i = 0; i < count; i++) { hbool_t is_compound_subset = FALSE; - size_t dense_dtype_size = 0; - /* Determine whether it's possible to send the data as a binary blob instead of as JSON */ if (H5T_NO_CLASS == (dtype_class = H5Tget_class(transfer_info[i].mem_type_id))) FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "a given memory datatype is invalid"); if ((is_variable_str = H5Tis_variable_str(transfer_info[i].mem_type_id)) < 0) FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "a given memory datatype is invalid"); + /* Only perform a binary transfer for fixed-length datatype datasets with an * All or Hyperslab selection. Point selections are dealt with by POSTing the * point list as JSON in the request body. */ - is_transfer_binary = (H5T_VLEN != dtype_class) && !is_variable_str; + is_transfer_binary = !is_variable_str; /* Follow the semantics for the use of H5S_ALL */ if (H5S_ALL == transfer_info[i].mem_space_id && H5S_ALL == transfer_info[i].file_space_id) { @@ -922,6 +922,11 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa */ transfer_info[i].mem_space_id = transfer_info[i].file_space_id = transfer_info[i].dataset->u.dataset.space_id; + + if (H5S_SEL_ERROR == (sel_type = H5Sget_select_type(transfer_info[i].file_space_id))) + FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace selection type"); + is_transfer_binary = is_transfer_binary && (H5S_SEL_POINTS != sel_type); + H5Sselect_all(transfer_info[i].file_space_id); } /* end if */ else if (H5S_ALL == transfer_info[i].file_space_id) { @@ -930,6 +935,11 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa * is set to the "all" selection. */ transfer_info[i].file_space_id = transfer_info[i].dataset->u.dataset.space_id; + + if (H5S_SEL_ERROR == (sel_type = H5Sget_select_type(transfer_info[i].file_space_id))) + FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace selection type"); + is_transfer_binary = is_transfer_binary && (H5S_SEL_POINTS != sel_type); + H5Sselect_all(transfer_info[i].file_space_id); } /* end if */ else { @@ -948,14 +958,18 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa "can't copy selection from file space to memory space"); } /* end if */ - /* Since the selection in the dataset's file dataspace is not set - * to "all", convert the selection into JSON */ - - /* Retrieve the selection type here for later use */ + /* Determine whether it's possible to send the data as a binary blob instead of as JSON */ if (H5S_SEL_ERROR == (sel_type = H5Sget_select_type(transfer_info[i].file_space_id))) FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get dataspace selection type"); + + /* Only perform a binary transfer for fixed-length datatype datasets with an + * All or Hyperslab selection. Point selections are dealt with by POSTing the + * point list as JSON in the request body. + */ is_transfer_binary = is_transfer_binary && (H5S_SEL_POINTS != sel_type); + /* Since the selection in the dataset's file dataspace is not set + * to "all", convert the selection into a string */ if (RV_convert_dataspace_selection_to_string(transfer_info[i].file_space_id, &selection_body, &selection_body_len, is_transfer_binary) < 0) FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTCONVERT, FAIL, @@ -976,17 +990,41 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa printf("-> %lld points selected in memory dataspace\n\n", mem_select_npoints); #endif + if ((file_type_size = H5Tget_size(transfer_info[i].file_type_id)) == 0) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "unable to get size of file datatype"); + + if ((mem_type_size = H5Tget_size(transfer_info[i].mem_type_id)) == 0) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "unable to get size of memory datatype"); + + mem_data_size = (size_t)file_select_npoints * mem_type_size; + + if ((is_write_contiguous = RV_dataspace_selection_is_contiguous(transfer_info[i].mem_space_id)) < 0) + FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, + "Unable to determine if the dataspace selection is contiguous"); + + if (!is_write_contiguous) { + if (NULL == (transfer_info[i].u.write_info.gather_buf = (char *)RV_malloc(mem_data_size))) + FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, + "can't allocate space for the 'write_body' values"); + if (H5Dgather(transfer_info[i].mem_space_id, transfer_info[i].u.write_info.uinfo.buffer, + transfer_info[i].mem_type_id, mem_data_size, + transfer_info[i].u.write_info.gather_buf, NULL, NULL) < 0) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't gather data to write buffer"); + transfer_info[i].u.write_info.uinfo.buffer = transfer_info[i].u.write_info.gather_buf; + } + /* Handle conversion from memory datatype to file datatype, if necessary */ if ((needs_tconv = RV_need_tconv(transfer_info[i].file_type_id, transfer_info[i].mem_type_id)) < 0) FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "unable to check if datatypes need conversion"); + dest_dtype_size = file_type_size; + if (needs_tconv) { #ifdef RV_CONNECTOR_DEBUG printf("-> Beginning type conversion for write\n"); #endif - RV_subset_t subset_type = H5T_SUBSET_BADVALUE; - hid_t dest_dtype = H5I_INVALID_HID; - size_t dest_dtype_size = 0; + RV_subset_t subset_type = H5T_SUBSET_BADVALUE; + hid_t dest_dtype = H5I_INVALID_HID; if ((file_type_size = H5Tget_size(transfer_info[i].file_type_id)) == 0) FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "unable to get size of file datatype"); @@ -1009,16 +1047,14 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa if ((H5Tpack(transfer_info[i].u.write_info.dense_cmpd_subset_dtype_id)) < 0) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTPACK, FAIL, "can't pack compound datatype"); - if ((dense_dtype_size = + if ((dest_dtype_size = H5Tget_size(transfer_info[i].u.write_info.dense_cmpd_subset_dtype_id)) == 0) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dense dtype size"); - dest_dtype = transfer_info[i].u.write_info.dense_cmpd_subset_dtype_id; - dest_dtype_size = dense_dtype_size; + dest_dtype = transfer_info[i].u.write_info.dense_cmpd_subset_dtype_id; } else { - dest_dtype = transfer_info[i].file_type_id; - dest_dtype_size = file_type_size; + dest_dtype = transfer_info[i].file_type_id; } /* Initialize type conversion */ @@ -1028,9 +1064,7 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa /* Copy memory to avoid modifying user-provided write buffer */ memset(transfer_info[i].tconv_buf, 0, file_type_size * (size_t)mem_select_npoints); - memcpy(transfer_info[i].tconv_buf, - (transfer_info[i].transfer_type == READ) ? transfer_info[i].u.read_info.buf - : transfer_info[i].u.write_info.buf, + memcpy(transfer_info[i].tconv_buf, transfer_info[i].u.write_info.uinfo.buffer, mem_type_size * (size_t)mem_select_npoints); /* Perform type conversion on values to write */ @@ -1041,62 +1075,53 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa H5P_DEFAULT) < 0) FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "failed to convert file datatype to memory datatype"); - } - /* If type conversion was performed, write from conversion buffer. Otherwise, write from transfer - * buffer for this transfer type */ - if (transfer_info[i].tconv_buf) { - buf_to_write = transfer_info[i].tconv_buf; - } - else { - buf_to_write = (transfer_info[i].transfer_type == READ) ? transfer_info[i].u.read_info.buf - : transfer_info[i].u.write_info.buf; + transfer_info[i].u.write_info.uinfo.buffer = transfer_info[i].tconv_buf; } /* Setup the size of the data being transferred and the data buffer itself (for non-simple * types like object references or variable length types) */ if ((H5T_REFERENCE != dtype_class) && (H5T_VLEN != dtype_class) && !is_variable_str) { - size_t write_dtype_size = 0; - - if (!is_compound_subset) { - if (0 == (write_dtype_size = H5Tget_size(transfer_info[i].file_type_id))) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "file datatype is invalid"); - } - else { - write_dtype_size = dense_dtype_size; - } - write_body_len = (size_t)file_select_npoints * write_dtype_size; + write_body_len = (size_t)file_select_npoints * dest_dtype_size; if ((is_write_contiguous = RV_dataspace_selection_is_contiguous(transfer_info[i].mem_space_id)) < 0) FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "Unable to determine if the dataspace selection is contiguous"); - if (!is_write_contiguous) { - if (NULL == (transfer_info[i].u.write_info.write_body = (char *)RV_malloc(write_body_len))) - FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, - "can't allocate space for the 'write_body' values"); - if (H5Dgather(transfer_info[i].mem_space_id, buf_to_write, transfer_info[i].file_type_id, - write_body_len, transfer_info[i].u.write_info.write_body, NULL, NULL) < 0) - FUNC_GOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't gather data to write buffer"); - buf_to_write = transfer_info[i].u.write_info.write_body; - } - else { + + /* Start write buffer at offset if write selection is contiguous */ + if (is_write_contiguous) { if ((offset = RV_convert_start_to_offset(transfer_info[i].mem_space_id)) < 0) FUNC_GOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "Unable to determine memory offset value"); - buf_to_write = (const void *)((const char *)buf_to_write + (size_t)offset * write_dtype_size); + + transfer_info[i].u.write_info.uinfo.buffer = + (const void *)((const char *)transfer_info[i].u.write_info.uinfo.buffer + + (size_t)offset * dest_dtype_size); } } /* end if */ else { + if (H5T_VLEN == dtype_class) { + /* Pack vlen data into single buffer for server */ + if (RV_pack_vlen_data((const hvl_t *)transfer_info[i].u.write_info.uinfo.buffer, + (size_t)file_select_npoints, mem_type_id[0], + (void **)&transfer_info[i].u.write_info.vlen_buf, &write_body_len) < 0) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, + "can't convert vlen data to a binary buffer"); + + transfer_info[i].u.write_info.uinfo.buffer = transfer_info[i].u.write_info.vlen_buf; + } + if (H5T_STD_REF_OBJ == transfer_info[i].file_type_id) { /* Convert the buffer of rest_obj_ref_t's to a binary buffer */ if (RV_convert_obj_refs_to_buffer( - (const rv_obj_ref_t *)buf_to_write, (size_t)file_select_npoints, - &(transfer_info[i].u.write_info.write_body), &write_body_len) < 0) + (const rv_obj_ref_t *)transfer_info[i].u.write_info.uinfo.buffer, + (size_t)file_select_npoints, (char **)&(transfer_info[i].u.write_info.serialize_buf), + &write_body_len) < 0) FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "can't convert object ref/s to ref string/s"); - buf_to_write = transfer_info[i].u.write_info.write_body; + transfer_info[i].u.write_info.uinfo.buffer = transfer_info[i].u.write_info.serialize_buf; } /* end if */ } /* end else */ @@ -1121,7 +1146,7 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa transfer_info[i].curl_headers, is_transfer_binary ? "Content-Type: application/octet-stream" : "Content-Type: application/json"); - has_selection_body = is_transfer_binary && selection_body && (H5S_SEL_POINTS != sel_type); + has_selection_in_url = is_transfer_binary && selection_body && (H5S_SEL_POINTS != sel_type); /* Redirect cURL from the base URL to "/datasets//value" to write the value out */ if (is_compound_subset) { @@ -1143,7 +1168,7 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa /* If using element selection and member selection, preface member query with '&' to join it to previous queries. Otherwise, it's the first query, so preface it with '?' */ - if (has_selection_body) { + if (has_selection_in_url) { cmpd_query_ptr[0] = '&'; cmpd_query_ptr++; } @@ -1202,8 +1227,8 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa if ((url_len = snprintf(transfer_info[i].request_url, URL_MAX_LENGTH, "%s/datasets/%s/value%s%s%s", transfer_info[i].dataset->domain->u.file.server_info.base_URL, - transfer_info[i].dataset->URI, has_selection_body ? "?select=" : "", - has_selection_body ? selection_body : "", + transfer_info[i].dataset->URI, has_selection_in_url ? "?select=" : "", + has_selection_in_url ? selection_body : "", is_compound_subset ? cmpd_query : "") < 0)) FUNC_GOTO_ERROR(H5E_DATASET, H5E_SYSERRSTR, FAIL, "snprintf error"); @@ -1234,30 +1259,29 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate temporary buffer for base64-encoded write buffer"); - if (RV_base64_encode(buf_to_write, write_body_len, - &(transfer_info[i].u.write_info.base64_encoded_values), &value_body_len) < 0) + if (RV_base64_encode(transfer_info[i].u.write_info.uinfo.buffer, write_body_len, + (char **)&transfer_info[i].u.write_info.base64_encoded_values, + &value_body_len) < 0) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "can't base64-encode write buffer"); #ifdef RV_CONNECTOR_DEBUG - printf("-> Base64-encoded data buffer: %s\n\n", + printf("-> Base64-encoded data buffer for dataset %zu: %s\n\n", i, transfer_info[i].u.write_info.base64_encoded_values); #endif - - if (transfer_info[i].u.write_info.write_body) { - RV_free(transfer_info[i].u.write_info.write_body); - transfer_info[i].u.write_info.write_body = NULL; - } + /* Copy encoded values into format string */ write_body_len = (strlen(fmt_string) - 4) + selection_body_len + value_body_len; - if (NULL == (transfer_info[i].u.write_info.write_body = RV_malloc(write_body_len + 1))) + if (NULL == (transfer_info[i].u.write_info.point_sel_buf = RV_malloc(write_body_len + 1))) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate space for write buffer"); if ((bytes_printed = - snprintf(transfer_info[i].u.write_info.write_body, write_body_len + 1, fmt_string, + snprintf(transfer_info[i].u.write_info.point_sel_buf, write_body_len + 1, fmt_string, selection_body, transfer_info[i].u.write_info.base64_encoded_values)) < 0) FUNC_GOTO_ERROR(H5E_DATASET, H5E_SYSERRSTR, FAIL, "snprintf error"); + transfer_info[i].u.write_info.uinfo.buffer = transfer_info[i].u.write_info.point_sel_buf; + #ifdef RV_CONNECTOR_DEBUG - printf("-> Write body: %s\n\n", transfer_info[i].u.write_info.write_body); + printf("-> Write body: %s\n\n", transfer_info[i].u.write_info.selection_buf); #endif if (bytes_printed >= write_body_len + 1) @@ -1270,22 +1294,18 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa #ifdef RV_CONNECTOR_DEBUG printf("-> Setup cURL to POST point list for dataset write\n\n"); #endif - } /* end if */ + } - transfer_info[i].u.write_info.uinfo.buffer = - is_transfer_binary ? buf_to_write : transfer_info[i].u.write_info.write_body; transfer_info[i].u.write_info.uinfo.buffer_size = write_body_len; transfer_info[i].u.write_info.uinfo.bytes_sent = 0; /* Check to make sure that the size of the write body can safely be cast to a curl_off_t */ if (sizeof(curl_off_t) < sizeof(size_t)) - ASSIGN_TO_SMALLER_SIZE(transfer_info[i].u.write_info.write_len, curl_off_t, write_body_len, - size_t) + ASSIGN_TO_SMALLER_SIZE(write_len, curl_off_t, write_body_len, size_t) else if (sizeof(curl_off_t) > sizeof(size_t)) - transfer_info[i].u.write_info.write_len = (curl_off_t)write_body_len; + write_len = (curl_off_t)write_body_len; else - ASSIGN_TO_SAME_SIZE_UNSIGNED_TO_SIGNED(transfer_info[i].u.write_info.write_len, curl_off_t, - write_body_len, size_t) + ASSIGN_TO_SAME_SIZE_UNSIGNED_TO_SIGNED(write_len, curl_off_t, write_body_len, size_t) if (CURLE_OK != curl_easy_setopt(transfer_info[i].curl_easy_handle, CURLOPT_UPLOAD, 1)) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set up cURL to make HTTP PUT request: %s", @@ -1295,8 +1315,8 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa &(transfer_info[i].u.write_info.uinfo))) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set cURL PUT data: %s", transfer_info[i].curl_err_buf); - if (CURLE_OK != curl_easy_setopt(transfer_info[i].curl_easy_handle, CURLOPT_INFILESIZE_LARGE, - transfer_info[i].u.write_info.write_len)) + if (CURLE_OK != + curl_easy_setopt(transfer_info[i].curl_easy_handle, CURLOPT_INFILESIZE_LARGE, write_len)) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set cURL PUT data size: %s", transfer_info[i].curl_err_buf); if (CURLE_OK != curl_easy_setopt(transfer_info[i].curl_easy_handle, CURLOPT_HTTPHEADER, @@ -1308,7 +1328,7 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set cURL request URL: %s", transfer_info[i].curl_err_buf); - if (transfer_info[i].u.write_info.write_len > 0) { + if (write_len > 0) { if (CURLM_OK != curl_multi_add_handle(curl_multi_handle, transfer_info[i].curl_easy_handle)) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't add cURL handle to multi handle: %s", transfer_info[i].curl_err_buf); @@ -1318,6 +1338,7 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa RV_free(selection_body); selection_body = NULL; } + } /* End iteration over dsets to write to */ #ifdef RV_CONNECTOR_DEBUG @@ -1348,13 +1369,57 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa curl_easy_cleanup(transfer_info[i].curl_easy_handle); } - RV_free(transfer_info[i].u.write_info.write_body); - RV_free(transfer_info[i].request_url); - RV_free(transfer_info[i].u.write_info.base64_encoded_values); - RV_free(transfer_info[i].resp_buffer.buffer); + if (transfer_info[i].tconv_buf) { + /* Clean up memory allocated by type conversion of vlen types */ + htri_t contains_vlen = FALSE; + + if (contains_vlen = H5Tdetect_class(transfer_info[i].file_type_id, H5T_VLEN) < 0) + FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "can't determine if datatype contains Vlen type"); + + if (contains_vlen > 0) { + hid_t type_conv_space = H5I_INVALID_HID; + + /* Type conversion buffer is packed, so create a 1D dataspace to describe its layout */ + if (mem_select_npoints = H5Sget_select_npoints(transfer_info[i].mem_space_id) < 0) + FUNC_DONE_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, + "can't get number of elements in file space"); + + if ((type_conv_space = H5Screate_simple(1, &mem_select_npoints, NULL)) < 0) + FUNC_DONE_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL, + "can't create simple dataspace for type conversion buffer"); + + if ((H5Treclaim(transfer_info[i].mem_type_id, type_conv_space, H5P_DEFAULT, + transfer_info[i].tconv_buf)) < 0) + FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, + "can't free type conversion buffer for vlen type"); + + if ((H5Sclose(type_conv_space)) < 0) + FUNC_DONE_ERROR(H5E_DATASPACE, H5E_CANTCLOSEOBJ, FAIL, + "can't close dataspace for type conversion buffer"); + } - if (transfer_info[i].tconv_buf) RV_free(transfer_info[i].tconv_buf); + transfer_info[i].tconv_buf = NULL; + } + + if (transfer_info[i].u.write_info.gather_buf) + RV_free(transfer_info[i].u.write_info.gather_buf); + + if (transfer_info[i].u.write_info.serialize_buf) + RV_free(transfer_info[i].u.write_info.serialize_buf); + + if (transfer_info[i].u.write_info.vlen_buf) + RV_free(transfer_info[i].u.write_info.vlen_buf); + + if (transfer_info[i].u.write_info.base64_encoded_values) + RV_free(transfer_info[i].u.write_info.base64_encoded_values); + + if (transfer_info[i].u.write_info.point_sel_buf) + RV_free(transfer_info[i].u.write_info.point_sel_buf); + + RV_free(transfer_info[i].request_url); + RV_free(transfer_info[i].resp_buffer.buffer); if (transfer_info[i].bkg_buf) RV_free(transfer_info[i].bkg_buf); @@ -1364,6 +1429,9 @@ RV_dataset_write(size_t count, void *dset[], hid_t mem_type_id[], hid_t _mem_spa if (transfer_info[i].u.write_info.dense_cmpd_subset_dtype_id != H5I_INVALID_HID) H5Tclose(transfer_info[i].u.write_info.dense_cmpd_subset_dtype_id); + + if (selection_body) + RV_free(selection_body); } curl_multi_cleanup(curl_multi_handle); @@ -3902,6 +3970,77 @@ RV_setup_dataset_create_request_body(void *parent_obj, const char *name, hid_t t return ret_value; } /* end RV_setup_dataset_create_request_body() */ +/*------------------------------------------------------------------------- + * Function: RV_convert_point_selection_to_binary + * + * Purpose: Given an HDF5 dataspace containing a point selection, + * formats the selection within the + * dataspace into either a binary representation. + * This is currently used only for point reads. + * + * The buffer handed back by this + * function by the caller, else memory will be leaked. + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Matthew Larson + * May, 2024 + */ +static herr_t +RV_convert_point_selection_to_binary(hid_t space_id, void **out_buf, size_t *out_buf_len) +{ + hssize_t num_points; + hsize_t *point_list; + size_t point_buf_size; + char *out_buf_curr_pos; + int ndims; + herr_t ret_value = SUCCEED; + +#ifdef RV_CONNECTOR_DEBUG + printf("-> Converting selection within dataspace to JSON\n\n"); +#endif + + if (!out_buf) + FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "dataspace output buffer was NULL"); + + if (H5I_DATASPACE != H5Iget_type(space_id)) + FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "not a dataspace"); + + if ((ndims = H5Sget_simple_extent_ndims(space_id)) < 0) + FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve dataspace dimensionality"); + if (!ndims) + FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "0-dimension dataspace specified"); + + if (H5S_SEL_POINTS != H5Sget_select_type(space_id)) { + FUNC_GOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "cannot convert non-points selection to binary"); + } + + /* Format the point selection as a binary buffer containing each coordinate index */ + if ((num_points = H5Sget_select_npoints(space_id)) < 0) + FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get number of selected points"); + + point_buf_size = (size_t)(ndims * num_points) * sizeof(hsize_t); + + if ((point_list = (hsize_t *)RV_calloc(point_buf_size)) == NULL) + FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate space for binary point list"); + + if (H5Sget_select_elem_pointlist(space_id, 0, (hsize_t)num_points, point_list) < 0) + FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve point list"); + +done: + + if (ret_value >= 0) { + *out_buf = (void *)point_list; + *out_buf_len = point_buf_size; + } + else { + if (point_list) + RV_free(point_list); + } + + return ret_value; +} /* end RV_convert_point_selection_to_binary*/ + /*------------------------------------------------------------------------- * Function: RV_convert_dataspace_selection_to_string * @@ -3921,7 +4060,7 @@ RV_setup_dataset_create_request_body(void *parent_obj, const char *name, hid_t t * since JSON can't be included in the request body in that * case. * - * When req_param is specified as FALSE, the seleciton is + * When req_param is specified as FALSE, the selection is * formatted as JSON so that it can be included in the request * body of a dataset read/write. This form is primarily used * for point selections and hyperslab selections where the @@ -3991,15 +4130,16 @@ RV_convert_dataspace_selection_to_string(hid_t space_id, char **selection_string break; case H5S_SEL_POINTS: - FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, - "point selections are unsupported as a HTTP request parameter"); - break; + /* This case is never used - point writes send selection in the body as JSON, + * and point reads send selection in the body as binary */ + + FUNC_GOTO_ERROR(H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, + "point selection not currently supported as a request parameter"); case H5S_SEL_HYPERSLABS: { #ifdef RV_CONNECTOR_DEBUG printf("-> Hyperslab selection\n\n"); #endif - /* Format the hyperslab selection according to the 'select' request/query parameter. * This is composed of N triplets, one for each dimension of the dataspace, and looks like: * @@ -4590,17 +4730,18 @@ RV_dataset_read_cb(hid_t mem_type_id, hid_t mem_space_id, hid_t file_type_id, hi void *obj_ref_buf = NULL; - H5S_sel_type sel_type = H5S_SEL_NONE; - void *json_buf = NULL; + H5S_sel_type sel_type = H5S_SEL_NONE; + void *vlen_buf = NULL; + void *tconv_buf = NULL; + void *bkg_buf = NULL; - void *tconv_buf = NULL; - void *bkg_buf = NULL; + size_t file_type_size = 0; + size_t mem_type_size = 0; + hbool_t needs_tconv = FALSE; + hbool_t fill_bkg = FALSE; - size_t file_type_size = 0; - size_t mem_type_size = 0; - hbool_t needs_tconv = FALSE; - RV_tconv_reuse_t reuse = RV_TCONV_REUSE_NONE; - hbool_t fill_bkg = FALSE; + /* Used to scatter read data into user buffer */ + struct response_read_info resp_info; RV_subset_t subset_type = H5T_SUBSET_BADVALUE; @@ -4610,7 +4751,7 @@ RV_dataset_read_cb(hid_t mem_type_id, hid_t mem_space_id, hid_t file_type_id, hi if ((is_variable_str = H5Tis_variable_str(mem_type_id)) < 0) FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "memory datatype is invalid"); - /* It was verified during setup that num selected point in memory space == num selected points in + /* It was verified during setup that num selected points in memory space == num selected points in * filespace */ if ((file_select_npoints = H5Sget_select_npoints(mem_space_id)) < 0) FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "memory dataspace is invalid"); @@ -4625,114 +4766,113 @@ RV_dataset_read_cb(hid_t mem_type_id, hid_t mem_space_id, hid_t file_type_id, hi mem_data_size = (size_t)file_select_npoints * mem_type_size; + if ((sel_type = H5Sget_select_type(file_space_id)) < 0) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get selection type for file space"); + if ((H5T_REFERENCE != dtype_class) && (H5T_VLEN != dtype_class) && !is_variable_str) { - /* Scatter the read data out to the supplied read buffer according to the - * mem_type_id and mem_space_id given */ - struct response_read_info resp_info; - resp_info.read_size = &mem_data_size; + resp_info.buffer = resp_buffer.buffer; + } + else { + if (H5T_VLEN == dtype_class) { + /* Variable length data returned in "packed" binary form */ + if (RV_unpack_vlen_data(resp_buffer.buffer, mem_type_id, (size_t)file_select_npoints, &vlen_buf) < + 0) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_PARSEERROR, FAIL, "can't unpack vlen data from server"); - if ((sel_type = H5Sget_select_type(file_space_id)) < 0) - FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get selection type for file space"); + resp_info.buffer = vlen_buf; + } + else if (H5T_STD_REF_OBJ == mem_type_id) { + /* Convert the received binary buffer into a buffer of rest_obj_ref_t's */ + if (RV_convert_buffer_to_obj_refs(resp_buffer.buffer, (size_t)file_select_npoints, + (rv_obj_ref_t **)&obj_ref_buf, &file_data_size) < 0) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, + "can't convert ref string/s to object ref array"); - if (sel_type != H5S_SEL_POINTS) { - resp_info.buffer = resp_buffer.buffer; + resp_info.buffer = obj_ref_buf; } else { - /* Server response is JSON instead of binary. - * Parse its 'value' field to a binary array to use for src_buf */ - if (RV_parse_response(resp_buffer.buffer, (void *)&file_type_id, (void *)&json_buf, - RV_json_values_to_binary_callback) < 0) - FUNC_GOTO_ERROR(H5E_DATASET, H5E_PARSEERROR, FAIL, "can't parse values"); - - resp_info.buffer = json_buf; + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unsupported datatype"); } + } - if ((needs_tconv = RV_need_tconv(file_type_id, mem_type_id)) < 0) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "unable to check if datatypes need conversion"); + if ((needs_tconv = RV_need_tconv(file_type_id, mem_type_id)) < 0) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "unable to check if datatypes need conversion"); - if (needs_tconv) { + if (needs_tconv) { #ifdef RV_CONNECTOR_DEBUG - printf("-> Beginning type conversion\n"); + printf("-> Beginning type conversion\n"); #endif - /* Initialize type conversion */ - RV_tconv_init(file_type_id, &file_type_size, mem_type_id, &mem_type_size, - (size_t)file_select_npoints, TRUE, FALSE, &tconv_buf, &bkg_buf, &reuse, &fill_bkg); - - /* Handle compound subsetting */ - if (RV_get_cmpd_subset_type(file_type_id, mem_type_id, &subset_type) < 0) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get compound type subset info"); - - if (subset_type < 0) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, - "error while checking if types are compound subsets"); + /* Initialize type conversion */ + RV_tconv_init(file_type_id, &file_type_size, mem_type_id, &mem_type_size, (size_t)file_select_npoints, + TRUE, FALSE, &tconv_buf, &bkg_buf, NULL, &fill_bkg); - if (subset_type == H5T_SUBSET_DST) { - /* Populate background buffer with bytes from user input buffer */ - if (!bkg_buf || !fill_bkg) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, - "compound subset requires bkg buffer"); + /* Handle compound subsetting */ + if (RV_get_cmpd_subset_type(file_type_id, mem_type_id, &subset_type) < 0) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get compound type subset info"); - memcpy(bkg_buf, buf, file_type_size * (size_t)file_select_npoints); - } + if (subset_type < 0) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, + "error while checking if types are compound subsets"); - /* Perform type conversion on response values */ - if (reuse == RV_TCONV_REUSE_TCONV) { - /* Use read buffer as type conversion buffer */ - if (H5Tconvert(file_type_id, mem_type_id, (size_t)file_select_npoints, resp_info.buffer, - bkg_buf, H5P_DEFAULT) < 0) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, - "failed to convert file datatype to memory datatype"); - } - else if (reuse == RV_TCONV_REUSE_BKG) { - /* Use read buffer as background buffer */ - memcpy(tconv_buf, resp_info.buffer, file_type_size * (size_t)file_select_npoints); + if (subset_type == H5T_SUBSET_DST) { + /* Populate background buffer with bytes from user input buffer */ + if (!bkg_buf || !fill_bkg) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "compound subset requires bkg buffer"); - if (H5Tconvert(file_type_id, mem_type_id, (size_t)file_select_npoints, tconv_buf, - resp_info.buffer, H5P_DEFAULT) < 0) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, - "failed to convert file datatype to memory datatype"); - resp_info.buffer = tconv_buf; - } - else { - /* Use newly allocated buffer for type conversion */ - memcpy(tconv_buf, resp_info.buffer, file_type_size * (size_t)file_select_npoints); + memcpy(bkg_buf, buf, file_type_size * (size_t)file_select_npoints); + } - if (H5Tconvert(file_type_id, mem_type_id, (size_t)file_select_npoints, tconv_buf, bkg_buf, - H5P_DEFAULT) < 0) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, - "failed to convert file datatype to memory datatype"); + /* Perform type conversion on response values */ + /* TODO - Reuse bkg buffer/read buffer if possible */ + memcpy(tconv_buf, resp_info.buffer, file_type_size * (size_t)file_select_npoints); - resp_info.buffer = tconv_buf; - } - } + if (H5Tconvert(file_type_id, mem_type_id, (size_t)file_select_npoints, tconv_buf, bkg_buf, + H5P_DEFAULT) < 0) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, + "failed to convert file datatype to memory datatype"); - if (H5Dscatter(dataset_read_scatter_op, &resp_info, mem_type_id, mem_space_id, buf) < 0) - FUNC_GOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't scatter data to read buffer"); + resp_info.buffer = tconv_buf; } - else { - if (H5T_STD_REF_OBJ == mem_type_id) { - /* Convert the received binary buffer into a buffer of rest_obj_ref_t's */ - if (RV_convert_buffer_to_obj_refs(resp_buffer.buffer, (size_t)file_select_npoints, - (rv_obj_ref_t **)&obj_ref_buf, &file_data_size) < 0) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, - "can't convert ref string/s to object ref array"); - memcpy(buf, obj_ref_buf, file_data_size); - } - else { - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unsupported datatype"); - } - } + /* Scatter the read data out to the supplied read buffer according to the + * mem_type_id and mem_space_id given */ + resp_info.read_size = &mem_data_size; + + if (H5Dscatter(dataset_read_scatter_op, &resp_info, mem_type_id, mem_space_id, buf) < 0) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't scatter data to read buffer"); done: if (obj_ref_buf) RV_free(obj_ref_buf); - if (json_buf) - RV_free(json_buf); + if (tconv_buf) { + htri_t contains_vlen = FALSE; + + if ((contains_vlen = H5Tdetect_class(mem_type_id, H5T_VLEN)) < 0) + FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't determine if datatype contains Vlen type"); + + if (contains_vlen > 0) { + hid_t type_conv_space = H5I_INVALID_HID; - if (tconv_buf) + /* Vlen buffer is packed, so create a 1D dataspace to describe its layout */ + if ((type_conv_space = H5Screate_simple(1, &file_select_npoints, NULL)) < 0) + FUNC_DONE_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL, + "can't create simple dataspace for unpacked vlen buffer"); + + if ((H5Treclaim(mem_type_id, type_conv_space, H5P_DEFAULT, vlen_buf)) < 0) + FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, + "can't free type conversion buffer for vlen type"); + + if (H5Sclose(type_conv_space) < 0) + FUNC_DONE_ERROR(H5E_DATASPACE, H5E_CANTCLOSEOBJ, FAIL, + "can't close dataspace for type conversion buffer"); + } RV_free(tconv_buf); + } + + if (vlen_buf) { + RV_free(vlen_buf); + } if (bkg_buf) RV_free(bkg_buf); @@ -4946,166 +5086,6 @@ RV_convert_start_to_offset(hid_t space_id) return ret_value; } /* end RV_convert_start_to_offset() */ -/*------------------------------------------------------------------------- - * Function: RV_json_values_to_binary_callback - * - * Purpose: A callback for RV_parse_response which will search - * an HTTP response for the "value" field, and extract - * the values into a newly allocated binary buffer. - * - * Expects data in to be a pointer to an hid_t for the - * datatype in the response, and data out to be the - * address of a pointer that will point to the - * newly allocated buffer. - * - * Return: Non-negative on success/Negative on failure - */ -static herr_t -RV_json_values_to_binary_callback(char *HTTP_response, const void *callback_data_in, void *callback_data_out) -{ - - void **out_buf = (void **)callback_data_out; - const hid_t dtype_id = *(const hid_t *)callback_data_in; - yajl_val parse_tree = NULL, key_obj; - char *parsed_string; - herr_t ret_value = SUCCEED; - void *value_buffer = NULL; - size_t dtype_size = 0; - H5T_class_t dtype_class = H5T_NO_CLASS; - -#ifdef RV_CONNECTOR_DEBUG - printf("-> Converting response JSON values to binary buffer\n\n"); -#endif - - if (!HTTP_response) - FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "HTTP response buffer was NULL"); - if (!out_buf) - FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "output buffer was NULL"); - - if ((dtype_size = H5Tget_size(dtype_id)) <= 0) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get datatype size"); - - if (NULL == (parse_tree = yajl_tree_parse(HTTP_response, NULL, 0))) - FUNC_GOTO_ERROR(H5E_OBJECT, H5E_PARSEERROR, FAIL, "parsing JSON failed"); - - /* Get the 'value' array */ - if (NULL == (key_obj = yajl_tree_get(parse_tree, value_keys, yajl_t_array))) - FUNC_GOTO_ERROR(H5E_OBJECT, H5E_PARSEERROR, FAIL, "unable to find 'value' key in JSON"); - - if (!YAJL_IS_ARRAY(key_obj)) - FUNC_GOTO_ERROR(H5E_OBJECT, H5E_PARSEERROR, FAIL, "parsed response is not an array of values"); - - if ((value_buffer = calloc(YAJL_GET_ARRAY(key_obj)->len, dtype_size)) == NULL) - FUNC_GOTO_ERROR(H5E_OBJECT, H5E_CANTALLOC, FAIL, "memory allocation failed for value buffer"); - - for (size_t i = 0; i < YAJL_GET_ARRAY(key_obj)->len; i++) { - yajl_val val = YAJL_GET_ARRAY(key_obj)->values[i]; - - if (RV_json_values_to_binary_recursive(val, dtype_id, - (void *)((char *)value_buffer + i * dtype_size)) < 0) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_PARSEERROR, FAIL, "failed to parse datatype from json"); - } - -done: - if (parse_tree) - yajl_tree_free(parse_tree); - - if (ret_value >= 0 && value_buffer) - *out_buf = value_buffer; - - if (ret_value < 0 && value_buffer) - RV_free(value_buffer); - - return ret_value; -} - -/* Helper function for RV_json_values_to_binary_callback */ -herr_t -RV_json_values_to_binary_recursive(yajl_val value_entry, hid_t dtype_id, void *value_buffer) -{ - herr_t ret_value = SUCCEED; - H5T_class_t dtype_class = H5T_NO_CLASS; - size_t dtype_size = 0; - - if ((dtype_size = H5Tget_size(dtype_id)) <= 0) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get datatype size"); - - if ((dtype_class = H5Tget_class(dtype_id)) == H5T_NO_CLASS) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get datatype class"); - - if (dtype_class == H5T_INTEGER) { - if (H5Tequal(dtype_id, H5T_NATIVE_INT) != TRUE) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, - "parsing non-native integer types is unsupported"); - - if (!YAJL_IS_INTEGER(value_entry)) - FUNC_GOTO_ERROR(H5E_OBJECT, H5E_PARSEERROR, FAIL, - "parsed yajl val has incorrect type; expected integer"); - - *((int *)value_buffer) = (int)YAJL_GET_INTEGER(value_entry); - } - else if (dtype_class == H5T_FLOAT) { - if (H5Tequal(dtype_id, H5T_NATIVE_FLOAT) == TRUE) { - if (!YAJL_IS_DOUBLE(value_entry)) - FUNC_GOTO_ERROR(H5E_OBJECT, H5E_PARSEERROR, FAIL, - "parsed yajl val has incorrect type; expected float-like"); - - *((float *)value_buffer) = (float)YAJL_GET_DOUBLE(value_entry); - } - else if (H5Tequal(dtype_id, H5T_NATIVE_DOUBLE) == TRUE) { - if (!YAJL_IS_DOUBLE(value_entry)) - FUNC_GOTO_ERROR(H5E_OBJECT, H5E_PARSEERROR, FAIL, - "parsed yajl val has incorrect type; expected double"); - - *((double *)value_buffer) = YAJL_GET_DOUBLE(value_entry); - } - else { - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, - "parsing non-native float types is unsupported"); - } - } - else if (dtype_class == H5T_COMPOUND) { - /* Recursively parse each member of the compound type */ - int nmembers = 0; - size_t offset = 0; - size_t member_size = 0; - hid_t member_dtype_id = H5I_INVALID_HID; - yajl_val member_val; - - if ((nmembers = H5Tget_nmembers(dtype_id)) < 0) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, - "can't get number of members in compound datatype"); - - for (int i = 0; i < nmembers; i++) { - if ((member_dtype_id = H5Tget_member_type(dtype_id, (unsigned int)i)) < 0) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, - "can't get datatype of member in compound datatype"); - - if ((member_size = H5Tget_size(member_dtype_id)) == 0) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get size of member datatype"); - - if ((member_val = YAJL_GET_OBJECT(value_entry)->values[i]) == NULL) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_PARSEERROR, FAIL, - "failed to parse member of compound type"); - - if (RV_json_values_to_binary_recursive(member_val, member_dtype_id, - (void *)((char *)value_buffer + offset)) < 0) - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_PARSEERROR, FAIL, "failed to parse member datatype"); - - offset += member_size; - member_val = NULL; - member_size = 0; - member_dtype_id = H5I_INVALID_HID; - } - } - else { - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unsupported datatype class for parsing"); - } - -done: - return ret_value; -} - /* Get the amount of compound type members in both destination and source type */ herr_t RV_get_cmpd_subset_nmembers(hid_t src_type_id, hid_t dst_type_id, size_t *num_cmpd_members) @@ -5217,6 +5197,192 @@ RV_get_cmpd_subset_nmembers(hid_t src_type_id, hid_t dst_type_id, size_t *num_cm if (dst_member_type > 0) if (H5Tclose(dst_member_type) < 0) FUNC_DONE_ERROR(H5E_DATATYPE, H5E_CANTCLOSEOBJ, FAIL, "can't close destination datatype member"); +} + +/*------------------------------------------------------------------------- + * Function: RV_pack_vlen_data + * + * Purpose: Converts a buffer of hvl_t instances to a buffer containing + * the length of sequence following by each sequence's data. + * + * Allocate memory under the *out pointer on success which must + * be freed by calling function. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Matthew Larson + * March, 2024 + */ +static herr_t +RV_pack_vlen_data(const hvl_t *in, size_t nelems, hid_t dtype_id, void **out, size_t *out_size) +{ + herr_t ret_value = SUCCEED; + char *out_buf = NULL; + char *out_buf_curr_pos = NULL; + size_t buf_size = 0; + size_t parent_dtype_size = 0; + size_t target_size = 0; + hid_t parent_dtype = H5I_INVALID_HID; + H5T_class_t parent_dtype_class = H5T_NO_CLASS; + + if (!in) + FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "given input buffer was NULL"); + + if (!out) + FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "given output buffer was NULL"); + + if (!out_size) + FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "given output size was NULL"); + + if (nelems == 0) + FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "given input buffer must contain > 0 elements"); + + if ((parent_dtype = H5Tget_super(dtype_id)) < 0) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get vlen parent datatype"); + + if ((parent_dtype_class = H5Tget_class(parent_dtype)) < 0) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "can't get class of vlen parent dtype"); + + /* Only simple vlen types are currently supported */ + if (parent_dtype_class != H5T_INTEGER && parent_dtype_class != H5T_FLOAT && + parent_dtype_class != H5T_STRING) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, + "vlen datatypes are only supported for int, float, and string base types"); + + if ((parent_dtype_size = H5Tget_size(parent_dtype)) == 0) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get size of vlen parent datatype"); + + if ((out_buf = calloc(DATASET_VLEN_JSON_BODY_DEFAULT_SIZE, 1)) == NULL) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate space for packed vlen data"); + + *out_size = DATASET_VLEN_JSON_BODY_DEFAULT_SIZE; + buf_size = *out_size; + out_buf_curr_pos = out_buf; + + /* Pack each variable length sequence into the buffer */ + for (size_t i = 0; i < nelems; i++) { + /* Make sure space exists for this sequence */ + ptrdiff_t curr_size = out_buf_curr_pos - out_buf; + hvl_t vl = in[i]; + uint64_t len_bytes = 0; + + target_size = (size_t)curr_size + sizeof(uint32_t) + vl.len * parent_dtype_size; + + CHECKED_REALLOC(out_buf, buf_size, target_size, out_buf_curr_pos, H5E_DATASET, FAIL); + + /* Copy sequence length and sequence contents into output buffer */ + len_bytes = vl.len * parent_dtype_size; + + /* Sequence length must fit into 4 bytes */ + if (len_bytes > UINT32_MAX) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "sequence exceeded maximum size"); + + memcpy(out_buf_curr_pos, (uint64_t *)&len_bytes, sizeof(uint32_t)); + out_buf_curr_pos += sizeof(uint32_t); + + memcpy(out_buf_curr_pos, vl.p, vl.len * parent_dtype_size); + out_buf_curr_pos += vl.len * parent_dtype_size; + *out_size = (size_t)(out_buf_curr_pos - out_buf); + } + + *out = out_buf; +done: + if (ret_value < 0 && out_buf) + RV_free(out_buf); + + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: RV_unpack_vlen_data + * + * Purpose: Converts a buffer of packed hvl_t instances to a buffer of + * hvl_t instances. + * + * Allocates memory for hvl_t instances which should be freed with + * H5Treclaim, and allocates under *out which should + * be freed directly by the calling function. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Matthew Larson + * March, 2024 + */ +static herr_t +RV_unpack_vlen_data(char *in, hid_t vlen_dtype_id, size_t nelems, void **out) +{ + herr_t ret_value = SUCCEED; + char *out_buf = NULL; + char *out_buf_curr_ptr; + char *in_buf_ptr = in; + hid_t parent_dtype = H5I_INVALID_HID; + size_t parent_dtype_size; + hvl_t *vl; + + if (!in) + FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "provided vlen datatype was NULL"); + + if (!out) + FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "provided output pointer was NULL"); + + if ((out_buf = RV_calloc(nelems * sizeof(hvl_t))) == NULL) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate space for unpacking vlen data"); + + if ((parent_dtype = H5Tget_super(vlen_dtype_id)) < 0) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get parent of vlen datatype"); + + if ((parent_dtype_size = H5Tget_size(parent_dtype)) == 0) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get size of vlen parent datatype"); + + for (size_t i = 0; i < nelems; i++) { + vl = ((hvl_t *)out_buf) + i; + uint32_t num_seq_bytes; + + memcpy(&num_seq_bytes, in_buf_ptr, sizeof(uint32_t)); + in_buf_ptr += sizeof(uint32_t); + + if (num_seq_bytes % parent_dtype_size != 0) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, + "vlen sequence length is not a multiple of element size"); + + vl->len = num_seq_bytes / parent_dtype_size; + + /* Allocate memory for and copy sequence data */ + if (vl->len > 0) { + if ((vl->p = RV_calloc(num_seq_bytes)) == NULL) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, + "can't allocate space for unpacking vlen data"); + + memcpy(vl->p, in_buf_ptr, num_seq_bytes); + } + else { + vl->p = NULL; + } + + in_buf_ptr += num_seq_bytes; + } + + *out = out_buf; + +done: + + if (ret_value < 0 && out_buf) { + hvl_t tmp_vl; + /* Free any memory allocated for individual sequences */ + for (size_t i = 0; i < nelems; i++) { + memcpy(&tmp_vl, out_buf + i * sizeof(hvl_t), sizeof(hvl_t)); + + if (tmp_vl.len > 0 && tmp_vl.p) { + RV_free(tmp_vl.p); + } + } + + RV_free(out_buf); + } + + if (parent_dtype != H5I_INVALID_HID) + if (H5Tclose(parent_dtype) < 0) + FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "can't close vlen parent datatype"); return ret_value; } \ No newline at end of file diff --git a/src/rest_vol_datatype.c b/src/rest_vol_datatype.c index 3ceffabe..b6bbceb6 100644 --- a/src/rest_vol_datatype.c +++ b/src/rest_vol_datatype.c @@ -659,8 +659,9 @@ RV_parse_datatype(char *type, hbool_t need_truncate) * Purpose: Given a datatype, this function creates a JSON-formatted * string representation of the datatype. * - * Can be called recursively for the case of Array and - * Compound Datatypes. The parameter 'nested' should always be + * Can be called recursively for the case of Array, + * Compound, and Variable-length datatypes. + * The parameter 'nested' should always be * supplied as FALSE, as the function itself handles the * correct passing of the parameter when processing nested * datatypes (such as the base type for an Array datatype). @@ -696,6 +697,7 @@ RV_convert_datatype_to_JSON(hid_t type_id, char **type_body, size_t *type_body_l char *array_base_type = NULL; char **compound_member_strings = NULL; char *compound_member_name = NULL; + char *vlen_base_type = NULL; char *out_string = NULL; char *out_string_curr_pos; /* The "current position" pointer used to print to the appropriate place in the buffer and not overwrite important leading data */ @@ -1311,7 +1313,45 @@ RV_convert_datatype_to_JSON(hid_t type_id, char **type_body, size_t *type_body_l } /* H5T_REFERENCE */ case H5T_VLEN: { - FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unsupported datatype - VLEN"); + size_t vlen_base_type_len = 0; + const char *const fmt_string = "{" + "\"class\": \"H5T_VLEN\"," + "\"base\": %s" + "}"; + + /* Get the class and name of the base datatype */ + if ((type_base_class = H5Tget_super(type_id)) < 0) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get base datatype for vlen sequence"); + + if (RV_convert_datatype_to_JSON(type_base_class, &vlen_base_type, &vlen_base_type_len, true, + server_version) < 0) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSERIALIZE, FAIL, + "can't convert parent of vlen type to JSON"); + + /* Check whether the buffer needs to be grown */ + bytes_to_print = vlen_base_type_len + (strlen(fmt_string) - 2) + 1; + + buf_ptrdiff = out_string_curr_pos - out_string; + + if (buf_ptrdiff < 0) + FUNC_GOTO_ERROR( + H5E_INTERNAL, H5E_BADVALUE, FAIL, + "unsafe cast: datatype buffer pointer different was negative - this should not happen!"); + + CHECKED_REALLOC(out_string, out_string_len, (size_t)buf_ptrdiff + bytes_to_print, + out_string_curr_pos, H5E_DATATYPE, FAIL); + + /* Build datatype body by appending the base type */ + if ((bytes_printed = snprintf(out_string_curr_pos, out_string_len - (size_t)buf_ptrdiff, + fmt_string, vlen_base_type)) < 0) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_SYSERRSTR, FAIL, "snprintf error"); + + if ((size_t)bytes_printed >= out_string_len - (size_t)buf_ptrdiff) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_SYSERRSTR, FAIL, + "vlen datatype string size exceeded allocated buffer size"); + + out_string_curr_pos += bytes_printed; + break; } /* H5T_VLEN */ @@ -1371,6 +1411,8 @@ RV_convert_datatype_to_JSON(hid_t type_id, char **type_body, size_t *type_body_l H5free_memory(enum_value_name); if (enum_mapping) RV_free(enum_mapping); + if (vlen_base_type) + RV_free(vlen_base_type); return ret_value; } /* end RV_convert_datatype_to_JSON() */ @@ -1432,13 +1474,15 @@ RV_convert_JSON_to_datatype(const char *type) hid_t datatype = FAIL; hid_t *compound_member_type_array = NULL; hid_t enum_base_type = FAIL; + hid_t ret_value = FAIL; + hid_t vlen_parent_type = H5I_INVALID_HID; + const char *path_name = NULL; char **compound_member_names = NULL; char *datatype_class = NULL; char *array_base_type_substring = NULL; char *tmp_cmpd_type_buffer = NULL; char *tmp_enum_base_type_buffer = NULL; - const char *path_name = NULL; - hid_t ret_value = FAIL; + char *tmp_vlen_type_buffer = NULL; #ifdef RV_CONNECTOR_DEBUG printf("-> Converting JSON buffer %s to hid_t\n", type); @@ -2068,6 +2112,76 @@ RV_convert_JSON_to_datatype(const char *type) else FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "invalid reference type"); } /* end if */ + else if (!strcmp(datatype_class, "H5T_VLEN")) { + ptrdiff_t buf_ptrdiff; + char *type_section_ptr = NULL; + char *section_start, *section_end; + size_t tmp_vlen_type_buffer_size = 0; + +#ifdef RV_CONNECTOR_DEBUG + printf("-> Variable-length Datatype\n"); +#endif + + /* Allocate temporary buffer for parent substring */ + tmp_vlen_type_buffer_size = DATATYPE_BODY_DEFAULT_SIZE; + + if (NULL == (tmp_vlen_type_buffer = (char *)RV_malloc(tmp_vlen_type_buffer_size))) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTALLOC, FAIL, + "can't allocate temporary buffer for storing type information"); + + /* Locate the beginning and end of the parent type's "type" section and copy that substring into an + * hid_t and store it for later insertion once the Variable-Length Datatype has been created. */ + + /* Start the search from the "base" JSON key */ + if (NULL == (type_section_ptr = strstr(type, "\"base\""))) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_PARSEERROR, FAIL, + "can't find \"base\" information section in datatype string"); + + /* Search for the initial '{' brace that begins the section */ + if (NULL == (section_start = strstr(type_section_ptr, "{"))) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_PARSEERROR, FAIL, + "can't find beginning '{' of \"base\" information section in datatype string " + "- misformatted JSON likely"); + + /* Continue forward through the string buffer character-by-character until the end of this JSON + * object section is found. + */ + FIND_JSON_SECTION_END(section_start, section_end, H5E_DATATYPE, FAIL); + + /* Check if the temporary buffer needs to grow to accommodate this "type" substring */ + buf_ptrdiff = section_end - type_section_ptr; + if (buf_ptrdiff < 0) + FUNC_GOTO_ERROR( + H5E_INTERNAL, H5E_BADVALUE, FAIL, + "unsafe cast: datatype buffer pointer difference was negative - this should not happen!"); + + CHECKED_REALLOC_NO_PTR(tmp_vlen_type_buffer, tmp_vlen_type_buffer_size, (size_t)buf_ptrdiff + 3, + H5E_DATATYPE, FAIL); + + /* Copy the "type" substring into the temporary buffer, wrapping it in enclosing braces to ensure + * that the string-to-datatype conversion function can correctly process the string + */ + memcpy(tmp_vlen_type_buffer + 1, type_section_ptr, (size_t)buf_ptrdiff); + tmp_vlen_type_buffer[0] = '{'; + tmp_vlen_type_buffer[(size_t)buf_ptrdiff + 1] = '}'; + tmp_vlen_type_buffer[(size_t)buf_ptrdiff + 2] = '\0'; + + /* Modify top-level key from 'base' to 'type' for string-to-datatype conversion function */ + memcpy(tmp_vlen_type_buffer + 2, type_class_keys[0], strlen(type_class_keys[0])); + +#ifdef RV_CONNECTOR_DEBUG + printf("-> Converting variable length datatype's parent type from JSON to hid_t\n", i); +#endif + + /* Recursively parse parent datatype from JSON */ + if ((vlen_parent_type = RV_convert_JSON_to_datatype(tmp_vlen_type_buffer)) < 0) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, + "can't convert vlen datatype parent from JSON representation"); + + /* Construct variable length type */ + if ((datatype = H5Tvlen_create(vlen_parent_type)) < 0) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTCREATE, FAIL, "can't create variable-length datatype"); + } else FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "unknown datatype class"); @@ -2105,6 +2219,8 @@ RV_convert_JSON_to_datatype(const char *type) RV_free(tmp_cmpd_type_buffer); if (tmp_enum_base_type_buffer) RV_free(tmp_enum_base_type_buffer); + if (tmp_vlen_type_buffer) + RV_free(tmp_vlen_type_buffer); if (FAIL != enum_base_type) if (H5Tclose(enum_base_type) < 0) FUNC_DONE_ERROR(H5E_DATATYPE, H5E_CANTCLOSEOBJ, FAIL, "can't close enum base datatype"); diff --git a/test/test_rest_vol.c b/test/test_rest_vol.c index 48baa70c..44825a2d 100644 --- a/test/test_rest_vol.c +++ b/test/test_rest_vol.c @@ -363,6 +363,10 @@ char filename[FILENAME_MAX_LENGTH]; #define DATASET_PROPERTY_LIST_TEST_DSET_NAME3 "property_list_test_dataset3" #define DATASET_PROPERTY_LIST_TEST_DSET_NAME4 "property_list_test_dataset4" +#define DATASET_VLEN_IO_TEST_EXTENT 50 +#define DATASET_VLEN_IO_TEST_NUM_POINTS 4 +#define DATASET_VLEN_IO_TEST_DSET_NAME "vlen_io_test_dset" + /***************************************************** * * * Plugin Datatype test defines * @@ -674,6 +678,7 @@ static int test_write_dataset_data_verification(void); static int test_dataset_set_extent(void); static int test_unused_dataset_API_calls(void); static int test_dataset_property_lists(void); +static int test_dataset_vlen_io(void); /* Committed Datatype interface tests */ static int test_create_committed_datatype(void); @@ -815,6 +820,7 @@ static int (*dataset_tests[])(void) = {test_create_dataset_under_root, test_dataset_set_extent, test_unused_dataset_API_calls, test_dataset_property_lists, + test_dataset_vlen_io, NULL}; static int (*type_tests[])(void) = {test_create_committed_datatype, @@ -9671,6 +9677,405 @@ test_dataset_property_lists(void) return 1; } +static int +test_dataset_vlen_io(void) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t dset_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t fspace_id = H5I_INVALID_HID; + hid_t dtype_id = H5I_INVALID_HID; + + hvl_t init[DATASET_VLEN_IO_TEST_EXTENT]; + hvl_t wbuf[DATASET_VLEN_IO_TEST_EXTENT]; + hvl_t rbuf[DATASET_VLEN_IO_TEST_EXTENT]; + + const hsize_t dims[] = {DATASET_VLEN_IO_TEST_EXTENT}; + const hsize_t points[] = {2, 3, 4, 5}; + + /* Hyperslab specification for elements at indices 2-5 */ + hssize_t start[1] = {2}; + hssize_t stride[1] = {2}; + hssize_t block[1] = {2}; + hssize_t count[1] = {2}; + + TESTING("Reading and writing variable-length data to a dataset"); + + if (H5rest_init() < 0) + TEST_ERROR + + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR + if (H5Pset_fapl_rest_vol(fapl_id) < 0) + TEST_ERROR + + if ((file_id = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR + + if ((fspace_id = H5Screate_simple(1, dims, NULL)) < 0) + TEST_ERROR + + if ((dtype_id = H5Tvlen_create(H5T_NATIVE_INT)) < 0) { + H5_FAILED(); + printf(" couldn't create vlen datatype\n"); + goto error; + } + + if ((dset_id = H5Dcreate2(file_id, DATASET_VLEN_IO_TEST_DSET_NAME, dtype_id, fspace_id, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT)) < 0) { + H5_FAILED(); + printf(" couldn't create vlen dataset\n"); + goto error; + } + + memset(wbuf, 0, sizeof(hvl_t) * DATASET_VLEN_IO_TEST_EXTENT); + memset(rbuf, 0, sizeof(hvl_t) * DATASET_VLEN_IO_TEST_EXTENT); + memset(init, 0, sizeof(hvl_t) * DATASET_VLEN_IO_TEST_EXTENT); + + /* Test read and write to entire dataset with vlen data */ + + /* Generate data and allocate memory */ + for (size_t i = 0; i < DATASET_VLEN_IO_TEST_EXTENT; i++) { + init[i].len = 0; + init[i].p = NULL; + + wbuf[i].len = i + 1; + wbuf[i].p = calloc(i + 1, sizeof(int)); + + for (size_t j = 0; j < i + 1; j++) { + ((int *)wbuf[i].p)[j] = (int)((i)*1000 + j * 10); + } + } + + if (H5Dwrite(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (const void *)wbuf) < 0) { + H5_FAILED(); + printf(" couldn't write to entire vlen dataset\n"); + goto error; + } + + /* Read back entire dataset */ + if (H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) { + H5_FAILED(); + printf(" couldn't read from entire vlen dataset\n"); + goto error; + } + + /* Verify data */ + for (size_t i = 0; i < DATASET_VLEN_IO_TEST_EXTENT; i++) { + if (rbuf[i].len != wbuf[i].len) { + H5_FAILED(); + printf(" Wrong length! Seq #%zu expected len %zu, got len %zu\n", i, wbuf[i].len, rbuf[i].len); + goto error; + } + + for (size_t j = 0; j < rbuf[i].len; j++) { + int actual = ((int *)rbuf[i].p)[j]; + int expected = ((int *)wbuf[i].p)[j]; + + if (actual != expected) { + H5_FAILED(); + printf(" Wrong value! Seq %zu elem %zu expected %d but got %d\n", i, j, expected, actual); + goto error; + } + } + } + + if (H5Treclaim(dtype_id, fspace_id, H5P_DEFAULT, rbuf) < 0) + TEST_ERROR + + memset(rbuf, 0, sizeof(hvl_t) * DATASET_VLEN_IO_TEST_EXTENT); + + /* Test read and write to hyperslab selections with vlen data */ + + /* Write to hyperslab, then read all */ + /* Reset dataset */ + if (H5Dwrite(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (const void *)init) < 0) { + H5_FAILED(); + printf(" couldn't write to entire vlen dataset\n"); + goto error; + } + + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + TEST_ERROR + + if (H5Dwrite(dset_id, dtype_id, H5S_ALL, fspace_id, H5P_DEFAULT, (const void *)wbuf) < 0) { + H5_FAILED(); + printf(" couldn't write to hyperslab selection in vlen dataset\n"); + goto error; + } + + if (H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) { + H5_FAILED(); + printf(" couldn't read from entire vlen dataset\n"); + goto error; + } + + for (size_t i = 0; i < DATASET_VLEN_IO_TEST_EXTENT; i++) { + if (i >= 2 && i <= 5) { + if (rbuf[i].len != wbuf[i].len) { + H5_FAILED(); + printf(" Wrong length! Seq #%zu expected len %zu, got len %zu\n", i, wbuf[i].len, + rbuf[i].len); + goto error; + } + + for (size_t j = 0; j < rbuf[i].len; j++) { + int actual = ((int *)rbuf[i].p)[j]; + int expected = ((int *)wbuf[i].p)[j]; + + if (actual != expected) { + H5_FAILED(); + printf(" Wrong value! Seq %zu elem %zu expected %d but got %d\n", i, j, expected, + actual); + goto error; + } + } + } + else { + if (rbuf[i].len != init[i].len) { + H5_FAILED(); + printf(" Wrong length for unread slab element!"); + goto error; + } + if (rbuf[i].p != init[i].p) { + H5_FAILED(); + printf(" Wrong pointer (non-NULL) for unread slab element!"); + goto error; + } + } + } + + if (H5Sselect_all(fspace_id) < 0) + TEST_ERROR + if (H5Treclaim(dtype_id, fspace_id, H5P_DEFAULT, rbuf) < 0) + TEST_ERROR + memset(rbuf, 0, sizeof(hvl_t) * DATASET_VLEN_IO_TEST_EXTENT); + + /* Write to all, then read from hyperslab */ + + if (H5Dwrite(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (const void *)wbuf) < 0) { + H5_FAILED(); + printf(" couldn't write to entire vlen dataset\n"); + goto error; + } + + if (H5Sselect_hyperslab(fspace_id, H5S_SELECT_SET, start, stride, count, block) < 0) + TEST_ERROR + + if (H5Dread(dset_id, dtype_id, H5S_ALL, fspace_id, H5P_DEFAULT, rbuf) < 0) { + H5_FAILED(); + printf(" couldn't read from hyperslab selection in vlen dataset\n"); + goto error; + } + + for (size_t i = 0; i < DATASET_VLEN_IO_TEST_EXTENT; i++) { + if (i >= 2 && i <= 5) { + if (rbuf[i].len != wbuf[i].len) { + H5_FAILED(); + printf(" Wrong length! Seq #%zu expected len %zu, got len %zu\n", i, wbuf[i].len, + rbuf[i].len); + goto error; + } + + for (size_t j = 0; j < rbuf[i].len; j++) { + int actual = ((int *)rbuf[i].p)[j]; + int expected = ((int *)wbuf[i].p)[j]; + + if (actual != expected) { + H5_FAILED(); + printf(" Wrong value! Seq %zu elem %zu expected %d but got %d\n", i, j, expected, + actual); + goto error; + } + } + } + else { + if (rbuf[i].len != init[i].len) { + H5_FAILED(); + printf(" Wrong length for unread slab element!"); + goto error; + } + if (rbuf[i].p != init[i].p) { + H5_FAILED(); + printf(" Wrong pointer (non-NULL) for unread slab element!"); + goto error; + } + } + } + + if (H5Sselect_all(fspace_id) < 0) + TEST_ERROR + if (H5Treclaim(dtype_id, fspace_id, H5P_DEFAULT, rbuf) < 0) + TEST_ERROR + memset(rbuf, 0, sizeof(hvl_t) * DATASET_VLEN_IO_TEST_EXTENT); + + /* Test read and write to point selections with vlen data */ + + /* Reset dataset */ + if (H5Dwrite(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (const void *)init) < 0) { + H5_FAILED(); + printf(" couldn't write to entire vlen dataset\n"); + goto error; + } + + if (H5Sselect_elements(fspace_id, H5S_SELECT_SET, DATASET_VLEN_IO_TEST_NUM_POINTS, points) < 0) + TEST_ERROR + + if (H5Dwrite(dset_id, dtype_id, H5S_ALL, fspace_id, H5P_DEFAULT, (const void *)wbuf) < 0) { + H5_FAILED(); + printf(" couldn't write to point selection in vlen dataset\n"); + goto error; + } + + /* Read back entire dataset */ + if (H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) { + H5_FAILED(); + printf(" couldn't read from entire vlen dataset\n"); + goto error; + } + + /* Verify data */ + for (size_t i = 0; i < DATASET_VLEN_IO_TEST_EXTENT; i++) { + if (i >= 2 && i <= 5) { + if (rbuf[i].len != wbuf[i].len) { + H5_FAILED(); + printf(" Wrong length! Seq #%zu expected len %zu, got len %zu\n", i, wbuf[i].len, + rbuf[i].len); + goto error; + } + + for (size_t j = 0; j < rbuf[i].len; j++) { + int actual = ((int *)rbuf[i].p)[j]; + int expected = ((int *)wbuf[i].p)[j]; + + if (actual != expected) { + H5_FAILED(); + printf(" Wrong value! Seq %zu elem %zu expected %d but got %d\n", i, j, expected, + actual); + goto error; + } + } + } + else { + if (rbuf[i].len != init[i].len) { + H5_FAILED(); + printf(" Wrong length for unread slab element!"); + goto error; + } + if (rbuf[i].p != init[i].p) { + H5_FAILED(); + printf(" Wrong pointer (non-NULL) for unread slab element!"); + goto error; + } + } + } + + if (H5Sselect_all(fspace_id) < 0) + TEST_ERROR + if (H5Treclaim(dtype_id, fspace_id, H5P_DEFAULT, rbuf) < 0) + TEST_ERROR + memset(rbuf, 0, sizeof(hvl_t) * DATASET_VLEN_IO_TEST_EXTENT); + + /* Test point read */ + + /* Write to entire dataset */ + if (H5Dwrite(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (const void *)wbuf) < 0) { + H5_FAILED(); + printf(" couldn't write to entire vlen dataset\n"); + goto error; + } + + /* Read back selected points */ + if (H5Sselect_elements(fspace_id, H5S_SELECT_SET, DATASET_VLEN_IO_TEST_NUM_POINTS, points) < 0) + TEST_ERROR + + if (H5Dread(dset_id, dtype_id, H5S_ALL, fspace_id, H5P_DEFAULT, rbuf) < 0) { + H5_FAILED(); + printf(" couldn't read from point selection in vlen dataset\n"); + goto error; + } + + /* Verify data */ + for (size_t i = 0; i < DATASET_VLEN_IO_TEST_EXTENT; i++) { + if (i >= 2 && i <= 5) { + if (rbuf[i].len != wbuf[i].len) { + H5_FAILED(); + printf(" Wrong length! Seq #%zu expected len %zu, got len %zu\n", i, wbuf[i].len, + rbuf[i].len); + goto error; + } + + for (size_t j = 0; j < rbuf[i].len; j++) { + int actual = ((int *)rbuf[i].p)[j]; + int expected = ((int *)wbuf[i].p)[j]; + + if (actual != expected) { + H5_FAILED(); + printf(" Wrong value! Seq %zu elem %zu expected %d but got %d\n", i, j, expected, + actual); + goto error; + } + } + } + else { + if (rbuf[i].len != init[i].len) { + H5_FAILED(); + printf(" Wrong length for unread slab element!"); + goto error; + } + if (rbuf[i].p != init[i].p) { + H5_FAILED(); + printf(" Wrong pointer (non-NULL) for unread slab element!"); + goto error; + } + } + } + + if (H5Sselect_all(fspace_id) < 0) + TEST_ERROR + if (H5Treclaim(dtype_id, fspace_id, H5P_DEFAULT, rbuf) < 0) + TEST_ERROR + if (H5Treclaim(dtype_id, fspace_id, H5P_DEFAULT, wbuf) < 0) + TEST_ERROR + + memset(wbuf, 0, sizeof(hvl_t) * DATASET_VLEN_IO_TEST_EXTENT); + memset(rbuf, 0, sizeof(hvl_t) * DATASET_VLEN_IO_TEST_EXTENT); + + if (H5Sclose(fspace_id) < 0) + TEST_ERROR + if (H5Dclose(dset_id) < 0) + TEST_ERROR + if (H5Pclose(fapl_id) < 0) + TEST_ERROR + if (H5Tclose(dtype_id) < 0) + TEST_ERROR + if (H5Fclose(file_id) < 0) + TEST_ERROR + if (H5rest_term() < 0) + TEST_ERROR + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY + { + H5Sselect_all(fspace_id); + H5Treclaim(dtype_id, fspace_id, H5P_DEFAULT, rbuf); + H5Treclaim(dtype_id, fspace_id, H5P_DEFAULT, wbuf); + H5Sclose(fspace_id); + H5Dclose(dset_id); + H5Pclose(fapl_id); + H5Tclose(dtype_id); + H5Fclose(file_id); + H5rest_term(); + } + H5E_END_TRY; + + return 1; +} + /***************************************************** * * * Plugin Committed Datatype tests *