From 1930a61102556d71c11e3237463e26871c576dad Mon Sep 17 00:00:00 2001 From: mattjala <124107509+mattjala@users.noreply.github.com> Date: Wed, 10 Apr 2024 13:12:21 -0500 Subject: [PATCH] Support H5Fget_filesize (#119) --- src/rest_vol.c | 61 ++++++++++++++++++- src/rest_vol.h | 7 ++- src/rest_vol_dataset.c | 4 +- src/rest_vol_debug.c | 24 ++++++++ src/rest_vol_debug.h | 1 + src/rest_vol_file.c | 134 ++++++++++++++++++++++++++++++++++++++++- src/rest_vol_file.h | 1 + 7 files changed, 223 insertions(+), 9 deletions(-) diff --git a/src/rest_vol.c b/src/rest_vol.c index dfd0a384..ceb3a57f 100644 --- a/src/rest_vol.c +++ b/src/rest_vol.c @@ -141,6 +141,10 @@ const char *attributes_keys[] = {"attributes", (const char *)0}; /* JSON keys to retrieve allocated size */ const char *allocated_size_keys[] = {"allocated_size", (const char *)0}; +/* JSON keys to retrieve information from a scan of a domain */ +const char *scan_info_keys[] = {"scan_info", (const char *)0}; +const char *allocated_bytes_keys[] = {"allocated_bytes", (const char *)0}; + /* JSON keys to retrieve objects accessed through path(s) */ const char *h5paths_keys[] = {"h5paths", (const char *)0}; @@ -256,7 +260,7 @@ static const H5VL_class_t H5VL_rest_g = { RV_file_open, RV_file_get, RV_file_specific, - NULL, + RV_file_optional, RV_file_close, }, @@ -3514,9 +3518,9 @@ RV_parse_server_version(char *HTTP_response, const void *callback_data_in, void return ret_value; } -/* Helper function to parse an object's allocated size from server response */ +/* Helper function to parse a non-domain object's allocated size from server response */ herr_t -RV_parse_allocated_size_callback(char *HTTP_response, const void *callback_data_in, void *callback_data_out) +RV_parse_allocated_size_cb(char *HTTP_response, void *callback_data_in, void *callback_data_out) { yajl_val parse_tree = NULL, key_obj = NULL; herr_t ret_value = SUCCEED; @@ -4391,3 +4395,54 @@ RV_curl_post(CURL *curl_handle, server_info_t *server_info, const char *request_ return ret_value; } +/*------------------------------------------------------------------------- + * Function: RV_parse_domain_allocated_size_cb + * + * Purpose: Given an HSDS response containing scan information about a + * domain, retrieves the allocated_byte values and modifies + * callback_data_out to point to it. HSDS returns scan info + * on a domain request with the 'verbose' parameter + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Matthew Larson + * January, 2024 + */ +herr_t +RV_parse_domain_allocated_size_cb(char *HTTP_response, const void *callback_data_in, void *callback_data_out) +{ + yajl_val parse_tree = NULL, key_obj; + char *parsed_object_string; + size_t *filesize = (size_t *)callback_data_out; + herr_t ret_value = SUCCEED; + +#ifdef RV_CONNECTOR_DEBUG + printf("-> Retrieving filesize from server's HTTP response\n\n"); +#endif + + if (!HTTP_response) + FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "HTTP response buffer was NULL"); + if (!filesize) + FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "output pointer was NULL"); + + if (NULL == (parse_tree = yajl_tree_parse(HTTP_response, NULL, 0))) + FUNC_GOTO_ERROR(H5E_CALLBACK, H5E_PARSEERROR, FAIL, "parsing JSON failed"); + + if (NULL == (key_obj = yajl_tree_get(parse_tree, scan_info_keys, yajl_t_object))) + FUNC_GOTO_ERROR(H5E_CALLBACK, H5E_PARSEERROR, FAIL, "couldn't get scan info"); + + if (NULL == (key_obj = yajl_tree_get(key_obj, allocated_bytes_keys, yajl_t_number))) { + FUNC_GOTO_ERROR(H5E_CALLBACK, H5E_PARSEERROR, FAIL, "couldn't parse allocated bytes"); + } + + if (YAJL_GET_INTEGER(key_obj) < 0) + FUNC_GOTO_ERROR(H5E_CALLBACK, H5E_PARSEERROR, FAIL, "parsed filesize is negative"); + + *filesize = (size_t)YAJL_GET_INTEGER(key_obj); + +done: + if (parse_tree) + yajl_tree_free(parse_tree); + + return ret_value; +} /* end RV_parse_domain_allocated_size_cb */ diff --git a/src/rest_vol.h b/src/rest_vol.h index d7ac9832..fb40652c 100644 --- a/src/rest_vol.h +++ b/src/rest_vol.h @@ -757,9 +757,10 @@ size_t H5_rest_curl_write_data_callback_no_global(char *buffer, size_t size, siz /* Helper to turn an object type into a string for a server request */ herr_t RV_set_object_type_header(H5I_type_t parent_obj_type, const char **parent_obj_type_header); -/* Helper function to parse an object's allocated size from server response */ -herr_t RV_parse_allocated_size_callback(char *HTTP_response, const void *callback_data_in, - void *callback_data_out); +/* Helper functions to parse an object's allocated size from server response */ +herr_t RV_parse_allocated_size_cb(char *HTTP_response, void *callback_data_in, void *callback_data_out); +herr_t RV_parse_domain_allocated_size_cb(char *HTTP_response, const void *callback_data_in, + void *callback_data_out); void RV_free_visited_link_hash_table_key(rv_hash_table_key_t value); diff --git a/src/rest_vol_dataset.c b/src/rest_vol_dataset.c index dfaa66aa..362889e8 100644 --- a/src/rest_vol_dataset.c +++ b/src/rest_vol_dataset.c @@ -1332,8 +1332,8 @@ RV_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **r dset->domain->u.file.filepath_name, CONTENT_TYPE_JSON) < 0) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset"); - if (RV_parse_allocated_size_callback(response_buffer.buffer, NULL, - args->args.get_storage_size.storage_size) < 0) + if (RV_parse_allocated_size_cb(response_buffer.buffer, NULL, + args->args.get_storage_size.storage_size) < 0) FUNC_GOTO_ERROR(H5E_DATASET, H5E_PARSEERROR, FAIL, "can't get allocated size from server response"); diff --git a/src/rest_vol_debug.c b/src/rest_vol_debug.c index cf40c24f..6ea75176 100644 --- a/src/rest_vol_debug.c +++ b/src/rest_vol_debug.c @@ -435,6 +435,30 @@ file_specific_type_to_string(H5VL_file_specific_t specific_type) } /* end switch */ } /* end file_specific_type_to_string() */ +/*------------------------------------------------------------------------- + * Function: file_optional_type_to_string + * + * Purpose: Helper function to convert each member of the + * H5VL_native_file_optional_args_t enum into its string representation + * + * Return: String representation of given object or '(unknown)' if + * the function can't determine the type of object it has + * been given (can't fail). + * + * Programmer: Matthew Larson + * January, 2024 + */ +const char * +file_optional_type_to_string(H5VL_file_optional_t optional_type) +{ + switch (optional_type) { + case H5VL_NATIVE_FILE_GET_SIZE: + return "RV_FILE_GET_FILESIZE"; + default: + return "(unknown)"; + } /* end switch */ +} /* end file_specific_type_to_string() */ + /*------------------------------------------------------------------------- * Function: group_get_type_to_string * diff --git a/src/rest_vol_debug.h b/src/rest_vol_debug.h index 117534d8..7e948ac8 100644 --- a/src/rest_vol_debug.h +++ b/src/rest_vol_debug.h @@ -32,6 +32,7 @@ const char *dataset_specific_type_to_string(H5VL_dataset_specific_t specific_typ const char *file_flags_to_string(unsigned flags); const char *file_get_type_to_string(H5VL_file_get_t get_type); const char *file_specific_type_to_string(H5VL_file_specific_t specific_type); +const char *file_optional_type_to_string(H5VL_file_optional_t optional_type); const char *group_get_type_to_string(H5VL_group_get_t get_type); const char *link_create_type_to_string(H5VL_link_create_t link_create_type); const char *link_get_type_to_string(H5VL_link_get_t get_type); diff --git a/src/rest_vol_file.c b/src/rest_vol_file.c index da07c5b9..9eabe251 100644 --- a/src/rest_vol_file.c +++ b/src/rest_vol_file.c @@ -32,6 +32,16 @@ struct get_obj_ids_udata_t { char *local_filename; } typedef get_obj_ids_udata_t; +/* Parameters for file 'optional' operations. + A subset of H5VL_native_file_optional_args_t */ +typedef union RV_file_optional_args_t { + /* H5VL_NATIVE_FILE_GET_SIZE */ + struct { + hsize_t *size; /* Size of file (OUT) */ + } get_size; + +} RV_file_optional_args_t; + /*------------------------------------------------------------------------- * Function: RV_file_create * @@ -677,6 +687,128 @@ RV_file_specific(void *obj, H5VL_file_specific_args_t *args, hid_t dxpl_id, void return ret_value; } /* end RV_file_specific() */ +/*------------------------------------------------------------------------- + * Function: RV_file_optional + * + * Purpose: Performs a connector-specific operation on an HDF5 file, such + * as calling the H5Fget_filesize routine + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Matthew Larson + * January, 2024 + */ +herr_t +RV_file_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_id, void **req) +{ + RV_object_t *file = (RV_object_t *)obj; + herr_t ret_value = SUCCEED; + size_t host_header_len = 0; + char *host_header = NULL; + char request_url[URL_MAX_LENGTH]; + int url_len = 0; + long http_response = 0; + +#ifdef RV_CONNECTOR_DEBUG + printf("-> Received file-optional call with following parameters:\n"); + printf(" - File-optional call type: %s\n", + file_optional_type_to_string(((H5VL_file_optional_t)args->op_type))); + if (file) { + printf(" - File's URI: %s\n", file->URI); + printf(" - File's pathname: %s\n", file->domain->u.file.filepath_name); + } /* end if */ + printf("\n"); +#endif + + switch (args->op_type) { + /* H5VL_FILE_GET_FILESIZE */ + case (H5VL_NATIVE_FILE_GET_SIZE): { + RV_file_optional_args_t *opt_args = (RV_file_optional_args_t *)args->args; + size_t *size_out = opt_args->get_size.size; + /* Setup cURL to make GET request */ + + /* Assemble URL */ + if ((url_len = snprintf(request_url, URL_MAX_LENGTH, "%s?verbose=1", + file->domain->u.file.server_info.base_URL)) < 0) + FUNC_GOTO_ERROR(H5E_FILE, H5E_SYSERRSTR, FAIL, "snprintf error"); + + if (url_len >= URL_MAX_LENGTH) + FUNC_GOTO_ERROR(H5E_FILE, H5E_SYSERRSTR, FAIL, + "H5Fget_filesize request URL size exceeded maximum URL size"); + + /* Setup the host header */ + host_header_len = strlen(file->domain->u.file.filepath_name) + strlen(host_string) + 1; + if (NULL == (host_header = (char *)RV_malloc(host_header_len))) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTALLOC, FAIL, + "can't allocate space for request Host header"); + + strcpy(host_header, host_string); + + curl_headers = + curl_slist_append(curl_headers, strncat(host_header, file->domain->u.file.filepath_name, + host_header_len - strlen(host_string) - 1)); + + /* Disable use of Expect: 100 Continue HTTP response */ + curl_headers = curl_slist_append(curl_headers, "Expect:"); + + if (CURLE_OK != + curl_easy_setopt(curl, CURLOPT_USERNAME, file->domain->u.file.server_info.username)) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't set cURL username: %s", curl_err_buf); + if (CURLE_OK != + curl_easy_setopt(curl, CURLOPT_PASSWORD, file->domain->u.file.server_info.password)) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't set cURL password: %s", curl_err_buf); + if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers)) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't set cURL HTTP headers: %s", curl_err_buf); + if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPGET, 1)) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't set up cURL to make HTTP GET request: %s", + curl_err_buf); + if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_URL, request_url)) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't set cURL request URL: %s", curl_err_buf); + +#ifdef RV_CONNECTOR_DEBUG + printf("-> Checking allocated bytes for domain using URL: %s\n\n", request_url); + + printf(" /**********************************\\\n"); + printf("-> | Making GET request to the server |\n"); + printf(" \\**********************************/\n\n"); +#endif + + CURL_PERFORM_NO_ERR(curl, FAIL); + + if (CURLE_OK != curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_response)) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get HTTP response code"); + + if (!(HTTP_SUCCESS(http_response))) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, + "request to server failed with HTTP response %ld", http_response); + + /* Retrieve number of bytes allocated for file from response */ + if (RV_parse_response(response_buffer.buffer, NULL, (void *)size_out, + RV_parse_domain_allocated_size_cb) < 0) + FUNC_GOTO_ERROR(H5E_FILE, H5E_PARSEERROR, FAIL, + "can't parse allocated bytes from server response"); + } + + break; + + default: + FUNC_GOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, FAIL, "unsupported optional file operation"); + break; + } + +done: + + if (host_header) + RV_free(host_header); + + if (curl_headers) { + curl_slist_free_all(curl_headers); + curl_headers = NULL; + } + + return (ret_value); +} /* end RV_file_optional */ + /*------------------------------------------------------------------------- * Function: RV_file_close * @@ -882,4 +1014,4 @@ RV_iterate_count_obj_cb(hid_t obj_id, void *udata) RV_free(containing_filename); return ret_value; -} +} \ No newline at end of file diff --git a/src/rest_vol_file.h b/src/rest_vol_file.h index 60b40dbf..bb148971 100644 --- a/src/rest_vol_file.h +++ b/src/rest_vol_file.h @@ -23,6 +23,7 @@ void *RV_file_create(const char *name, unsigned flags, hid_t fcpl_id, hid_t fap void *RV_file_open(const char *name, unsigned flags, hid_t fapl_id, hid_t dxpl_id, void **req); herr_t RV_file_get(void *obj, H5VL_file_get_args_t *args, hid_t dxpl_id, void **req); herr_t RV_file_specific(void *obj, H5VL_file_specific_args_t *args, hid_t dxpl_id, void **req); +herr_t RV_file_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_id, void **req); herr_t RV_file_close(void *file, hid_t dxpl_id, void **req); #ifdef __cplusplus