From 58c25069afedc7904a2e44e0199606fc14d937c2 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Wed, 2 Aug 2023 14:16:20 -0500 Subject: [PATCH 1/2] Implement support for fill values --- src/rest_vol.h | 6 ++- src/rest_vol_dataset.c | 87 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 85 insertions(+), 8 deletions(-) diff --git a/src/rest_vol.h b/src/rest_vol.h index fef8ca2c..50167ab9 100644 --- a/src/rest_vol.h +++ b/src/rest_vol.h @@ -312,7 +312,7 @@ H5VL_CAP_FLAG_OBJECT_MORE | H5VL_CAP_FLAG_CREATION_ORDER | H5VL_CAP_FLAG_ITERATE | \ H5VL_CAP_FLAG_BY_IDX | H5VL_CAP_FLAG_GET_PLIST | H5VL_CAP_FLAG_EXTERNAL_LINKS | \ H5VL_CAP_FLAG_HARD_LINKS | H5VL_CAP_FLAG_SOFT_LINKS | H5VL_CAP_FLAG_TRACK_TIMES | \ - H5VL_CAP_FLAG_FILTERS + H5VL_CAP_FLAG_FILTERS | H5VL_CAP_FLAG_FILL_VALUES /********************************** * * * Global Variables * @@ -630,7 +630,9 @@ void RV_free_visited_link_hash_table_key(rv_hash_table_key_t value); #define SERVER_VERSION_MATCHES_OR_EXCEEDS(version, major_needed, minor_needed, patch_needed) \ (version.major > major_needed) || (version.major == major_needed && version.minor > minor_needed) || \ - (version.major == major_needed && version.minor == minor_needed && version.patch >= patch_needed) + (version.major == major_needed && version.minor == minor_needed && version.patch >= patch_needed) \ + +#define SERVER_VERISON_SUPPORTS_FILL_VALUE_ENCODING(version) (SERVER_VERSION_MATCHES_OR_EXCEEDS(version, 0, 8, 1)) #ifdef __cplusplus } diff --git a/src/rest_vol_dataset.c b/src/rest_vol_dataset.c index ce17d67b..7eb118c8 100644 --- a/src/rest_vol_dataset.c +++ b/src/rest_vol_dataset.c @@ -26,8 +26,9 @@ static herr_t RV_setup_dataset_create_request_body(void *parent_obj, const char hid_t space_id, hid_t lcpl_id, hid_t dcpl, char **create_request_body, size_t *create_request_body_len); + static herr_t RV_convert_dataset_creation_properties_to_JSON(hid_t dcpl_id, char **creation_properties_body, - size_t *creation_properties_body_len); + size_t *creation_properties_body_len, hid_t type_id, server_api_version version); /* Helper function to convert a selection within an HDF5 Dataspace into a JSON-format string */ static herr_t RV_convert_dataspace_selection_to_string(hid_t space_id, char **selection_string, @@ -1837,7 +1838,7 @@ RV_parse_dataset_creation_properties_callback(char *HTTP_response, void *callbac */ static herr_t RV_convert_dataset_creation_properties_to_JSON(hid_t dcpl, char **creation_properties_body, - size_t *creation_properties_body_len) + size_t *creation_properties_body_len, hid_t type_id, server_api_version version) { const char *const leading_string = "\"creationProperties\": {"; H5D_alloc_time_t alloc_time; @@ -1850,6 +1851,9 @@ RV_convert_dataset_creation_properties_to_JSON(hid_t dcpl, char **creation_prope char *out_string_curr_pos; /* The "current position" pointer used to print to the appropriate place in the buffer and not overwrite important leading data */ int bytes_printed = 0; + void *fill_value = NULL; + char *encode_buf_out = NULL; + char *fill_value_str = NULL; herr_t ret_value = SUCCEED; #ifdef RV_CONNECTOR_DEBUG @@ -2082,6 +2086,8 @@ RV_convert_dataset_creation_properties_to_JSON(hid_t dcpl, char **creation_prope ******************************************************************/ { H5D_fill_value_t fill_status; + size_t fill_value_size = 0; + size_t encode_buf_out_size = 0; if (H5Pfill_value_defined(dcpl, &fill_status) < 0) FUNC_GOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve the \"fill value defined\" status"); @@ -2105,9 +2111,69 @@ RV_convert_dataset_creation_properties_to_JSON(hid_t dcpl, char **creation_prope strncat(out_string_curr_pos, null_value, null_value_len); out_string_curr_pos += null_value_len; - } /* end if */ - else { - FUNC_GOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "dataset fill values are unsupported"); + } else if (H5D_FILL_VALUE_USER_DEFINED == fill_status) { + if (!(SERVER_VERISON_SUPPORTS_FILL_VALUE_ENCODING(version))) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "server API version %zu.%zu.%zu does not support fill value encoding\n", version.major, version.minor, version.patch); + + if ((fill_value_size = H5Tget_size(type_id)) == 0) + FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get the size of fill value type"); + + if ((fill_value = RV_malloc(fill_value_size)) == NULL) + FUNC_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate space for fill value"); + + if (H5Pget_fill_value(dcpl, type_id, fill_value) < 0) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "can't get fill value from creation properties"); + + if (RV_base64_encode(fill_value, fill_value_size, &encode_buf_out, &encode_buf_out_size) < 0) + FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "can't base64-encode fill value"); + + /* Add encoded fill value to request body */ + size_t fill_value_str_len = strlen(", \"fillValue\": ") + strlen("\"") + strlen(encode_buf_out) + strlen("\""); + + if ((fill_value_str = RV_calloc(fill_value_str_len + 1)) == NULL) + FUNC_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate space for fill value string in request body"); + + snprintf(fill_value_str, fill_value_str_len + 1, "%s%s%s%s", + ", \"fillValue\": ", + "\"", + encode_buf_out, + "\""); + + /* Check whether the buffer needs to be grown */ + bytes_to_print = fill_value_str_len; + + buf_ptrdiff = out_string_curr_pos - out_string; + + if (buf_ptrdiff < 0) + FUNC_GOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, + "unsafe cast: dataset creation properties buffer pointer difference was " + "negative - this should not happen!"); + + CHECKED_REALLOC(out_string, out_string_len, (size_t)buf_ptrdiff + bytes_to_print, + out_string_curr_pos, H5E_DATASET, FAIL); + + strncat(out_string, fill_value_str, fill_value_str_len); + out_string_curr_pos += fill_value_str_len; + + /* Write the encoding used to the request body */ + const char *encoding_property = ", \"fillValue_encoding\": \"base64\""; + + bytes_to_print = strlen(encoding_property); + + buf_ptrdiff = out_string_curr_pos - out_string; + if (buf_ptrdiff < 0) + FUNC_GOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, + "unsafe cast: dataset creation properties buffer pointer difference was " + "negative - this should not happen!"); + + CHECKED_REALLOC(out_string, out_string_len, (size_t)buf_ptrdiff + bytes_to_print, + out_string_curr_pos, H5E_DATASET, FAIL); + + strncat(out_string, encoding_property, strlen(encoding_property)); + out_string_curr_pos += strlen(encoding_property); + + } else { + FUNC_GOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve the \"fill value defined\" status"); } /* end else */ } /* end if */ } @@ -2865,6 +2931,15 @@ RV_convert_dataset_creation_properties_to_JSON(hid_t dcpl, char **creation_prope if (chunk_dims_string) RV_free(chunk_dims_string); + if (fill_value) + RV_free(fill_value); + + if (encode_buf_out) + RV_free(encode_buf_out); + + if (fill_value_str) + RV_free(fill_value_str); + return ret_value; } /* end RV_convert_dataset_creation_properties_to_JSON() */ @@ -2936,7 +3011,7 @@ RV_setup_dataset_create_request_body(void *parent_obj, const char *name, hid_t t "layout H5D_CONTIGUOUS is unsupported for server versions before 0.8.0"); if (RV_convert_dataset_creation_properties_to_JSON(dcpl, &creation_properties_body, - &creation_properties_body_len) < 0) + &creation_properties_body_len, type_id, pobj->domain->u.file.server_version) < 0) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "can't convert Dataset Creation Properties to JSON representation"); } From 50f2d90456545735c17800435f0e981b65bc21e4 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Aug 2023 19:18:15 +0000 Subject: [PATCH 2/2] Committing clang-format changes --- src/rest_vol.h | 5 ++-- src/rest_vol_dataset.c | 61 +++++++++++++++++++++++------------------- 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/src/rest_vol.h b/src/rest_vol.h index 50167ab9..7c2ed479 100644 --- a/src/rest_vol.h +++ b/src/rest_vol.h @@ -630,9 +630,10 @@ void RV_free_visited_link_hash_table_key(rv_hash_table_key_t value); #define SERVER_VERSION_MATCHES_OR_EXCEEDS(version, major_needed, minor_needed, patch_needed) \ (version.major > major_needed) || (version.major == major_needed && version.minor > minor_needed) || \ - (version.major == major_needed && version.minor == minor_needed && version.patch >= patch_needed) \ + (version.major == major_needed && version.minor == minor_needed && version.patch >= patch_needed) -#define SERVER_VERISON_SUPPORTS_FILL_VALUE_ENCODING(version) (SERVER_VERSION_MATCHES_OR_EXCEEDS(version, 0, 8, 1)) +#define SERVER_VERISON_SUPPORTS_FILL_VALUE_ENCODING(version) \ + (SERVER_VERSION_MATCHES_OR_EXCEEDS(version, 0, 8, 1)) #ifdef __cplusplus } diff --git a/src/rest_vol_dataset.c b/src/rest_vol_dataset.c index 7eb118c8..0c5d44d8 100644 --- a/src/rest_vol_dataset.c +++ b/src/rest_vol_dataset.c @@ -28,7 +28,8 @@ static herr_t RV_setup_dataset_create_request_body(void *parent_obj, const char size_t *create_request_body_len); static herr_t RV_convert_dataset_creation_properties_to_JSON(hid_t dcpl_id, char **creation_properties_body, - size_t *creation_properties_body_len, hid_t type_id, server_api_version version); + size_t *creation_properties_body_len, + hid_t type_id, server_api_version version); /* Helper function to convert a selection within an HDF5 Dataspace into a JSON-format string */ static herr_t RV_convert_dataspace_selection_to_string(hid_t space_id, char **selection_string, @@ -1838,7 +1839,8 @@ RV_parse_dataset_creation_properties_callback(char *HTTP_response, void *callbac */ static herr_t RV_convert_dataset_creation_properties_to_JSON(hid_t dcpl, char **creation_properties_body, - size_t *creation_properties_body_len, hid_t type_id, server_api_version version) + size_t *creation_properties_body_len, hid_t type_id, + server_api_version version) { const char *const leading_string = "\"creationProperties\": {"; H5D_alloc_time_t alloc_time; @@ -1850,11 +1852,11 @@ RV_convert_dataset_creation_properties_to_JSON(hid_t dcpl, char **creation_prope char *out_string = NULL; char *out_string_curr_pos; /* The "current position" pointer used to print to the appropriate place in the buffer and not overwrite important leading data */ - int bytes_printed = 0; - void *fill_value = NULL; - char *encode_buf_out = NULL; - char *fill_value_str = NULL; - herr_t ret_value = SUCCEED; + int bytes_printed = 0; + void *fill_value = NULL; + char *encode_buf_out = NULL; + char *fill_value_str = NULL; + herr_t ret_value = SUCCEED; #ifdef RV_CONNECTOR_DEBUG printf("-> Converting dataset creation properties from DCPL to JSON\n\n"); @@ -2086,8 +2088,8 @@ RV_convert_dataset_creation_properties_to_JSON(hid_t dcpl, char **creation_prope ******************************************************************/ { H5D_fill_value_t fill_status; - size_t fill_value_size = 0; - size_t encode_buf_out_size = 0; + size_t fill_value_size = 0; + size_t encode_buf_out_size = 0; if (H5Pfill_value_defined(dcpl, &fill_status) < 0) FUNC_GOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve the \"fill value defined\" status"); @@ -2111,36 +2113,39 @@ RV_convert_dataset_creation_properties_to_JSON(hid_t dcpl, char **creation_prope strncat(out_string_curr_pos, null_value, null_value_len); out_string_curr_pos += null_value_len; - } else if (H5D_FILL_VALUE_USER_DEFINED == fill_status) { + } + else if (H5D_FILL_VALUE_USER_DEFINED == fill_status) { if (!(SERVER_VERISON_SUPPORTS_FILL_VALUE_ENCODING(version))) - FUNC_GOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "server API version %zu.%zu.%zu does not support fill value encoding\n", version.major, version.minor, version.patch); + FUNC_GOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, + "server API version %zu.%zu.%zu does not support fill value encoding\n", + version.major, version.minor, version.patch); if ((fill_value_size = H5Tget_size(type_id)) == 0) FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get the size of fill value type"); if ((fill_value = RV_malloc(fill_value_size)) == NULL) - FUNC_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate space for fill value"); + FUNC_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate space for fill value"); if (H5Pget_fill_value(dcpl, type_id, fill_value) < 0) - FUNC_GOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "can't get fill value from creation properties"); - + FUNC_GOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, + "can't get fill value from creation properties"); + if (RV_base64_encode(fill_value, fill_value_size, &encode_buf_out, &encode_buf_out_size) < 0) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "can't base64-encode fill value"); - + /* Add encoded fill value to request body */ - size_t fill_value_str_len = strlen(", \"fillValue\": ") + strlen("\"") + strlen(encode_buf_out) + strlen("\""); + size_t fill_value_str_len = + strlen(", \"fillValue\": ") + strlen("\"") + strlen(encode_buf_out) + strlen("\""); if ((fill_value_str = RV_calloc(fill_value_str_len + 1)) == NULL) - FUNC_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate space for fill value string in request body"); + FUNC_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "can't allocate space for fill value string in request body"); - snprintf(fill_value_str, fill_value_str_len + 1, "%s%s%s%s", - ", \"fillValue\": ", - "\"", - encode_buf_out, - "\""); + snprintf(fill_value_str, fill_value_str_len + 1, "%s%s%s%s", ", \"fillValue\": ", "\"", + encode_buf_out, "\""); /* Check whether the buffer needs to be grown */ - bytes_to_print = fill_value_str_len; + bytes_to_print = fill_value_str_len + 1; buf_ptrdiff = out_string_curr_pos - out_string; @@ -2171,9 +2176,10 @@ RV_convert_dataset_creation_properties_to_JSON(hid_t dcpl, char **creation_prope strncat(out_string, encoding_property, strlen(encoding_property)); out_string_curr_pos += strlen(encoding_property); - - } else { - FUNC_GOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve the \"fill value defined\" status"); + } + else { + FUNC_GOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, + "can't retrieve the \"fill value defined\" status"); } /* end else */ } /* end if */ } @@ -3011,7 +3017,8 @@ RV_setup_dataset_create_request_body(void *parent_obj, const char *name, hid_t t "layout H5D_CONTIGUOUS is unsupported for server versions before 0.8.0"); if (RV_convert_dataset_creation_properties_to_JSON(dcpl, &creation_properties_body, - &creation_properties_body_len, type_id, pobj->domain->u.file.server_version) < 0) + &creation_properties_body_len, type_id, + pobj->domain->u.file.server_version) < 0) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "can't convert Dataset Creation Properties to JSON representation"); }