From 2d9eb893c49067bb131a5272db7ff39dc62b4234 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Mon, 29 Jan 2024 14:24:47 -0600 Subject: [PATCH 1/2] Sync with master --- src/rest_vol.c | 61 ++++++++++++++++++- src/rest_vol.h | 10 ++- src/rest_vol_dataset.c | 4 +- src/rest_vol_debug.c | 24 ++++++++ src/rest_vol_debug.h | 1 + src/rest_vol_file.c | 134 ++++++++++++++++++++++++++++++++++++++++- src/rest_vol_file.h | 1 + 7 files changed, 226 insertions(+), 9 deletions(-) diff --git a/src/rest_vol.c b/src/rest_vol.c index 1c3a8fb0..68065fec 100644 --- a/src/rest_vol.c +++ b/src/rest_vol.c @@ -141,6 +141,10 @@ const char *attributes_keys[] = {"attributes", (const char *)0}; /* JSON keys to retrieve allocated size */ const char *allocated_size_keys[] = {"allocated_size", (const char *)0}; +/* JSON keys to retrieve information from a scan of a domain */ +const char *scan_info_keys[] = {"scan_info", (const char *)0}; +const char *allocated_bytes_keys[] = {"allocated_bytes", (const char *)0}; + /* Default size for the buffer to allocate during base64-encoding if the caller * of RV_base64_encode supplies a 0-sized buffer. */ @@ -253,7 +257,7 @@ static const H5VL_class_t H5VL_rest_g = { RV_file_open, RV_file_get, RV_file_specific, - NULL, + RV_file_optional, RV_file_close, }, @@ -3376,9 +3380,9 @@ RV_parse_server_version(char *HTTP_response, const void *callback_data_in, void return ret_value; } -/* Helper function to parse an object's allocated size from server response */ +/* Helper function to parse a non-domain object's allocated size from server response */ herr_t -RV_parse_allocated_size_callback(char *HTTP_response, const void *callback_data_in, void *callback_data_out) +RV_parse_allocated_size_cb(char *HTTP_response, void *callback_data_in, void *callback_data_out) { yajl_val parse_tree = NULL, key_obj = NULL; herr_t ret_value = SUCCEED; @@ -4250,3 +4254,54 @@ RV_curl_post(CURL *curl_handle, server_info_t *server_info, const char *request_ return ret_value; } +/*------------------------------------------------------------------------- + * Function: RV_parse_domain_allocated_size_cb + * + * Purpose: Given an HSDS response containing scan information about a + * domain, retrieves the allocated_byte values and modifies + * callback_data_out to point to it. HSDS returns scan info + * on a domain request with the 'verbose' parameter + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Matthew Larson + * January, 2024 + */ +herr_t +RV_parse_domain_allocated_size_cb(char *HTTP_response, const void *callback_data_in, void *callback_data_out) +{ + yajl_val parse_tree = NULL, key_obj; + char *parsed_object_string; + size_t *filesize = (size_t *)callback_data_out; + herr_t ret_value = SUCCEED; + +#ifdef RV_CONNECTOR_DEBUG + printf("-> Retrieving filesize from server's HTTP response\n\n"); +#endif + + if (!HTTP_response) + FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "HTTP response buffer was NULL"); + if (!filesize) + FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "output pointer was NULL"); + + if (NULL == (parse_tree = yajl_tree_parse(HTTP_response, NULL, 0))) + FUNC_GOTO_ERROR(H5E_CALLBACK, H5E_PARSEERROR, FAIL, "parsing JSON failed"); + + if (NULL == (key_obj = yajl_tree_get(parse_tree, scan_info_keys, yajl_t_object))) + FUNC_GOTO_ERROR(H5E_CALLBACK, H5E_PARSEERROR, FAIL, "couldn't get scan info"); + + if (NULL == (key_obj = yajl_tree_get(key_obj, allocated_bytes_keys, yajl_t_number))) { + FUNC_GOTO_ERROR(H5E_CALLBACK, H5E_PARSEERROR, FAIL, "couldn't parse allocated bytes"); + } + + if (YAJL_GET_INTEGER(key_obj) < 0) + FUNC_GOTO_ERROR(H5E_CALLBACK, H5E_PARSEERROR, FAIL, "parsed filesize is negative"); + + *filesize = (size_t)YAJL_GET_INTEGER(key_obj); + +done: + if (parse_tree) + yajl_tree_free(parse_tree); + + return ret_value; +} /* end RV_parse_domain_allocated_size_cb */ diff --git a/src/rest_vol.h b/src/rest_vol.h index 2407b30f..c3e27b3b 100644 --- a/src/rest_vol.h +++ b/src/rest_vol.h @@ -434,6 +434,9 @@ typedef struct RV_type_info { #define H5I_MAX_NUM_TYPES TYPE_MASK extern RV_type_info *RV_type_info_array_g[]; +/* Enum values in RV_file_optional_args_t that specify certain operations */ +#define RV_FILE_GET_SIZE 8 + /************************** * * * Typedefs * @@ -751,9 +754,10 @@ size_t H5_rest_curl_write_data_callback_no_global(char *buffer, size_t size, siz /* Helper to turn an object type into a string for a server request */ herr_t RV_set_object_type_header(H5I_type_t parent_obj_type, const char **parent_obj_type_header); -/* Helper function to parse an object's allocated size from server response */ -herr_t RV_parse_allocated_size_callback(char *HTTP_response, const void *callback_data_in, - void *callback_data_out); +/* Helper functions to parse an object's allocated size from server response */ +herr_t RV_parse_allocated_size_cb(char *HTTP_response, void *callback_data_in, void *callback_data_out); +herr_t RV_parse_domain_allocated_size_cb(char *HTTP_response, const void *callback_data_in, + void *callback_data_out); void RV_free_visited_link_hash_table_key(rv_hash_table_key_t value); diff --git a/src/rest_vol_dataset.c b/src/rest_vol_dataset.c index 4dc13371..8fde1465 100644 --- a/src/rest_vol_dataset.c +++ b/src/rest_vol_dataset.c @@ -1340,8 +1340,8 @@ RV_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **r dset->domain->u.file.filepath_name, CONTENT_TYPE_JSON) < 0) FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset"); - if (RV_parse_allocated_size_callback(response_buffer.buffer, NULL, - args->args.get_storage_size.storage_size) < 0) + if (RV_parse_allocated_size_cb(response_buffer.buffer, NULL, + args->args.get_storage_size.storage_size) < 0) FUNC_GOTO_ERROR(H5E_DATASET, H5E_PARSEERROR, FAIL, "can't get allocated size from server response"); diff --git a/src/rest_vol_debug.c b/src/rest_vol_debug.c index cf40c24f..ba1d0fdf 100644 --- a/src/rest_vol_debug.c +++ b/src/rest_vol_debug.c @@ -435,6 +435,30 @@ file_specific_type_to_string(H5VL_file_specific_t specific_type) } /* end switch */ } /* end file_specific_type_to_string() */ +/*------------------------------------------------------------------------- + * Function: file_optional_type_to_string + * + * Purpose: Helper function to convert each member of the + * H5VL_file_optional_t enum into its string representation + * + * Return: String representation of given object or '(unknown)' if + * the function can't determine the type of object it has + * been given (can't fail). + * + * Programmer: Matthew Larson + * January, 2024 + */ +const char * +file_optional_type_to_string(H5VL_optional_args_t optional_type) +{ + switch (optional_type.op_type) { + case RV_FILE_GET_SIZE: + return "RV_FILE_GET_FILESIZE"; + default: + return "(unknown)"; + } /* end switch */ +} /* end file_specific_type_to_string() */ + /*------------------------------------------------------------------------- * Function: group_get_type_to_string * diff --git a/src/rest_vol_debug.h b/src/rest_vol_debug.h index 117534d8..3d3118f1 100644 --- a/src/rest_vol_debug.h +++ b/src/rest_vol_debug.h @@ -32,6 +32,7 @@ const char *dataset_specific_type_to_string(H5VL_dataset_specific_t specific_typ const char *file_flags_to_string(unsigned flags); const char *file_get_type_to_string(H5VL_file_get_t get_type); const char *file_specific_type_to_string(H5VL_file_specific_t specific_type); +const char *file_optional_type_to_string(H5VL_optional_args_t optional_type); const char *group_get_type_to_string(H5VL_group_get_t get_type); const char *link_create_type_to_string(H5VL_link_create_t link_create_type); const char *link_get_type_to_string(H5VL_link_get_t get_type); diff --git a/src/rest_vol_file.c b/src/rest_vol_file.c index da07c5b9..6e02496c 100644 --- a/src/rest_vol_file.c +++ b/src/rest_vol_file.c @@ -32,6 +32,16 @@ struct get_obj_ids_udata_t { char *local_filename; } typedef get_obj_ids_udata_t; +/* Parameters for file 'optional' operations. + A subset of H5VL_native_file_optional_args_t */ +typedef union RV_file_optional_args_t { + /* H5VL_NATIVE_FILE_GET_SIZE */ + struct { + hsize_t *size; /* Size of file (OUT) */ + } get_size; + +} RV_file_optional_args_t; + /*------------------------------------------------------------------------- * Function: RV_file_create * @@ -677,6 +687,128 @@ RV_file_specific(void *obj, H5VL_file_specific_args_t *args, hid_t dxpl_id, void return ret_value; } /* end RV_file_specific() */ +/*------------------------------------------------------------------------- + * Function: RV_file_optional + * + * Purpose: Performs a connector-specific operation on an HDF5 file, such + * as calling the H5Fget_filesize routine + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Matthew Larson + * January, 2024 + */ +herr_t +RV_file_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_id, void **req) +{ + RV_object_t *file = (RV_object_t *)obj; + herr_t ret_value = SUCCEED; + size_t host_header_len = 0; + char *host_header = NULL; + char request_url[URL_MAX_LENGTH]; + int url_len = 0; + long http_response = 0; + +#ifdef RV_CONNECTOR_DEBUG + printf("-> Received file-optional call with following parameters:\n"); + printf(" - File-optional call type: %s\n", + file_optional_type_to_string(*((H5VL_optional_args_t *)args))); + if (file) { + printf(" - File's URI: %s\n", file->URI); + printf(" - File's pathname: %s\n", file->domain->u.file.filepath_name); + } /* end if */ + printf("\n"); +#endif + + switch (args->op_type) { + /* H5VL_FILE_GET_FILESIZE */ + case (RV_FILE_GET_SIZE): { + RV_file_optional_args_t *opt_args = (RV_file_optional_args_t *)args->args; + size_t *size_out = opt_args->get_size.size; + /* Setup cURL to make GET request */ + + /* Assemble URL */ + if ((url_len = snprintf(request_url, URL_MAX_LENGTH, "%s?verbose=1", + file->domain->u.file.server_info.base_URL)) < 0) + FUNC_GOTO_ERROR(H5E_FILE, H5E_SYSERRSTR, FAIL, "snprintf error"); + + if (url_len >= URL_MAX_LENGTH) + FUNC_GOTO_ERROR(H5E_FILE, H5E_SYSERRSTR, FAIL, + "H5Fget_filesize request URL size exceeded maximum URL size"); + + /* Setup the host header */ + host_header_len = strlen(file->domain->u.file.filepath_name) + strlen(host_string) + 1; + if (NULL == (host_header = (char *)RV_malloc(host_header_len))) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTALLOC, FAIL, + "can't allocate space for request Host header"); + + strcpy(host_header, host_string); + + curl_headers = + curl_slist_append(curl_headers, strncat(host_header, file->domain->u.file.filepath_name, + host_header_len - strlen(host_string) - 1)); + + /* Disable use of Expect: 100 Continue HTTP response */ + curl_headers = curl_slist_append(curl_headers, "Expect:"); + + if (CURLE_OK != + curl_easy_setopt(curl, CURLOPT_USERNAME, file->domain->u.file.server_info.username)) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't set cURL username: %s", curl_err_buf); + if (CURLE_OK != + curl_easy_setopt(curl, CURLOPT_PASSWORD, file->domain->u.file.server_info.password)) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't set cURL password: %s", curl_err_buf); + if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers)) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't set cURL HTTP headers: %s", curl_err_buf); + if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPGET, 1)) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't set up cURL to make HTTP GET request: %s", + curl_err_buf); + if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_URL, request_url)) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't set cURL request URL: %s", curl_err_buf); + +#ifdef RV_CONNECTOR_DEBUG + printf("-> Checking allocated bytes for domain using URL: %s\n\n", request_url); + + printf(" /**********************************\\\n"); + printf("-> | Making GET request to the server |\n"); + printf(" \\**********************************/\n\n"); +#endif + + CURL_PERFORM_NO_ERR(curl, FAIL); + + if (CURLE_OK != curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_response)) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get HTTP response code"); + + if (!(HTTP_SUCCESS(http_response))) + FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, + "request to server failed with HTTP response %ld", http_response); + + /* Retrieve number of bytes allocated for file from response */ + if (RV_parse_response(response_buffer.buffer, NULL, (void *)size_out, + RV_parse_domain_allocated_size_cb) < 0) + FUNC_GOTO_ERROR(H5E_FILE, H5E_PARSEERROR, FAIL, + "can't parse allocated bytes from server response"); + } + + break; + + default: + FUNC_GOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, FAIL, "unsupported optional file operation"); + break; + } + +done: + + if (host_header) + RV_free(host_header); + + if (curl_headers) { + curl_slist_free_all(curl_headers); + curl_headers = NULL; + } + + return (ret_value); +} /* end RV_file_optional */ + /*------------------------------------------------------------------------- * Function: RV_file_close * @@ -882,4 +1014,4 @@ RV_iterate_count_obj_cb(hid_t obj_id, void *udata) RV_free(containing_filename); return ret_value; -} +} \ No newline at end of file diff --git a/src/rest_vol_file.h b/src/rest_vol_file.h index 60b40dbf..bb148971 100644 --- a/src/rest_vol_file.h +++ b/src/rest_vol_file.h @@ -23,6 +23,7 @@ void *RV_file_create(const char *name, unsigned flags, hid_t fcpl_id, hid_t fap void *RV_file_open(const char *name, unsigned flags, hid_t fapl_id, hid_t dxpl_id, void **req); herr_t RV_file_get(void *obj, H5VL_file_get_args_t *args, hid_t dxpl_id, void **req); herr_t RV_file_specific(void *obj, H5VL_file_specific_args_t *args, hid_t dxpl_id, void **req); +herr_t RV_file_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_id, void **req); herr_t RV_file_close(void *file, hid_t dxpl_id, void **req); #ifdef __cplusplus From f30770be84551cba159358905eb8983cbb7092a6 Mon Sep 17 00:00:00 2001 From: Matthew Larson Date: Tue, 9 Apr 2024 17:11:09 -0500 Subject: [PATCH 2/2] Use native VOL enum type for optional file ops --- src/rest_vol.h | 3 --- src/rest_vol_debug.c | 8 ++++---- src/rest_vol_debug.h | 2 +- src/rest_vol_file.c | 4 ++-- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/rest_vol.h b/src/rest_vol.h index f348c740..fb40652c 100644 --- a/src/rest_vol.h +++ b/src/rest_vol.h @@ -440,9 +440,6 @@ typedef struct RV_type_info { #define H5I_MAX_NUM_TYPES TYPE_MASK extern RV_type_info *RV_type_info_array_g[]; -/* Enum values in RV_file_optional_args_t that specify certain operations */ -#define RV_FILE_GET_SIZE 8 - /************************** * * * Typedefs * diff --git a/src/rest_vol_debug.c b/src/rest_vol_debug.c index ba1d0fdf..6ea75176 100644 --- a/src/rest_vol_debug.c +++ b/src/rest_vol_debug.c @@ -439,7 +439,7 @@ file_specific_type_to_string(H5VL_file_specific_t specific_type) * Function: file_optional_type_to_string * * Purpose: Helper function to convert each member of the - * H5VL_file_optional_t enum into its string representation + * H5VL_native_file_optional_args_t enum into its string representation * * Return: String representation of given object or '(unknown)' if * the function can't determine the type of object it has @@ -449,10 +449,10 @@ file_specific_type_to_string(H5VL_file_specific_t specific_type) * January, 2024 */ const char * -file_optional_type_to_string(H5VL_optional_args_t optional_type) +file_optional_type_to_string(H5VL_file_optional_t optional_type) { - switch (optional_type.op_type) { - case RV_FILE_GET_SIZE: + switch (optional_type) { + case H5VL_NATIVE_FILE_GET_SIZE: return "RV_FILE_GET_FILESIZE"; default: return "(unknown)"; diff --git a/src/rest_vol_debug.h b/src/rest_vol_debug.h index 3d3118f1..7e948ac8 100644 --- a/src/rest_vol_debug.h +++ b/src/rest_vol_debug.h @@ -32,7 +32,7 @@ const char *dataset_specific_type_to_string(H5VL_dataset_specific_t specific_typ const char *file_flags_to_string(unsigned flags); const char *file_get_type_to_string(H5VL_file_get_t get_type); const char *file_specific_type_to_string(H5VL_file_specific_t specific_type); -const char *file_optional_type_to_string(H5VL_optional_args_t optional_type); +const char *file_optional_type_to_string(H5VL_file_optional_t optional_type); const char *group_get_type_to_string(H5VL_group_get_t get_type); const char *link_create_type_to_string(H5VL_link_create_t link_create_type); const char *link_get_type_to_string(H5VL_link_get_t get_type); diff --git a/src/rest_vol_file.c b/src/rest_vol_file.c index 6e02496c..9eabe251 100644 --- a/src/rest_vol_file.c +++ b/src/rest_vol_file.c @@ -712,7 +712,7 @@ RV_file_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_id, void **re #ifdef RV_CONNECTOR_DEBUG printf("-> Received file-optional call with following parameters:\n"); printf(" - File-optional call type: %s\n", - file_optional_type_to_string(*((H5VL_optional_args_t *)args))); + file_optional_type_to_string(((H5VL_file_optional_t)args->op_type))); if (file) { printf(" - File's URI: %s\n", file->URI); printf(" - File's pathname: %s\n", file->domain->u.file.filepath_name); @@ -722,7 +722,7 @@ RV_file_optional(void *obj, H5VL_optional_args_t *args, hid_t dxpl_id, void **re switch (args->op_type) { /* H5VL_FILE_GET_FILESIZE */ - case (RV_FILE_GET_SIZE): { + case (H5VL_NATIVE_FILE_GET_SIZE): { RV_file_optional_args_t *opt_args = (RV_file_optional_args_t *)args->args; size_t *size_out = opt_args->get_size.size; /* Setup cURL to make GET request */