Skip to content

Commit

Permalink
Support H5Dset_extent
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattjala
mattjala committed Oct 2, 2023
1 parent c2ab7ab commit a680f50
Show file tree
Hide file tree
Showing 7 changed files with 274 additions and 6 deletions.
15 changes: 15 additions & 0 deletions src/rest_vol.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ typedef struct H5_rest_ad_info_t {
hbool_t unattended;
} H5_rest_ad_info_t;

/* Global array containing information about open objects */
RV_type_info *RV_type_info_array_g[H5I_MAX_NUM_TYPES];

/* Host header string for specifying the host (Domain) for requests */
const char *const host_string = "X-Hdf-domain: ";

Expand Down Expand Up @@ -451,6 +454,12 @@ H5_rest_init(hid_t vipl_id)
curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
#endif

/* Set up global type info array */
for (size_t i = 0; i < H5I_MAX_NUM_TYPES; i++) {
RV_type_info_array_g[i] = RV_calloc(sizeof(RV_type_info));
RV_type_info_array_g[i]->table = rv_hash_table_new(rv_hash_string, H5_rest_compare_string_keys);
}

/* Register the connector with HDF5's error reporting API */
if ((H5_rest_err_class_g = H5Eregister_class(HDF5_VOL_REST_ERR_CLS_NAME, HDF5_VOL_REST_LIB_NAME,
HDF5_VOL_REST_LIB_VER)) < 0)
Expand Down Expand Up @@ -616,6 +625,12 @@ H5_rest_term(void)
curl_global_cleanup();
} /* end if */

/* Cleanup type info array */
for (size_t i = 0; i < H5I_MAX_NUM_TYPES; i++) {
rv_hash_table_free(RV_type_info_array_g[i]->table);
RV_free(RV_type_info_array_g[i]);
RV_type_info_array_g[i] = NULL;
}
/*
* "Forget" connector ID. This should normally be called by the library
* when it is closing the id, so no need to close it here.
Expand Down
12 changes: 12 additions & 0 deletions src/rest_vol.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,18 @@ struct response_buffer {
};
extern struct response_buffer response_buffer;

/* Struct containing information about open objects of each type in the VOL*/
typedef struct RV_type_info {
size_t open_count;
rv_hash_table_t *table;
} RV_type_info;

/* TODO - This is copied directly from the library */
#define TYPE_BITS 7
#define TYPE_MASK (((hid_t)1 << TYPE_BITS) - 1)
#define H5I_MAX_NUM_TYPES TYPE_MASK
extern RV_type_info *RV_type_info_array_g[];

/**************************
* *
* Typedefs *
Expand Down
11 changes: 11 additions & 0 deletions src/rest_vol_attr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,10 @@ RV_attr_create(void *obj, const H5VL_loc_params_t *loc_params, const char *attr_
printf("-> Created attribute\n\n");
#endif

if (rv_hash_table_insert(RV_type_info_array_g[H5I_ATTR]->table, (char *)new_attribute,
(char *)new_attribute) == 0)
FUNC_GOTO_ERROR(H5E_ATTR, H5E_CANTALLOC, NULL, "Failed to add attribute to type info array");

ret_value = (void *)new_attribute;

done:
Expand Down Expand Up @@ -752,6 +756,10 @@ RV_attr_open(void *obj, const H5VL_loc_params_t *loc_params, const char *attr_na
if ((attribute->u.attribute.acpl_id = H5Pcreate(H5P_ATTRIBUTE_CREATE)) < 0)
FUNC_GOTO_ERROR(H5E_PLIST, H5E_CANTCREATE, NULL, "can't create ACPL for attribute");

if (rv_hash_table_insert(RV_type_info_array_g[H5I_ATTR]->table, (char *)attribute, (char *)attribute) ==
0)
FUNC_GOTO_ERROR(H5E_ATTR, H5E_CANTALLOC, NULL, "Failed to add attribute to type info array");

ret_value = (void *)attribute;

done:
Expand Down Expand Up @@ -2737,6 +2745,9 @@ RV_attr_close(void *attr, hid_t dxpl_id, void **req)
FUNC_DONE_ERROR(H5E_PLIST, H5E_CANTCLOSEOBJ, FAIL, "can't close ACPL");
} /* end if */

if (RV_type_info_array_g[H5I_ATTR])
rv_hash_table_remove(RV_type_info_array_g[H5I_ATTR]->table, (char *)_attr);

if (RV_file_close(_attr->domain, H5P_DEFAULT, NULL) < 0)
FUNC_GOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "couldn't close attr domain");

Expand Down
209 changes: 205 additions & 4 deletions src/rest_vol_dataset.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,10 @@ RV_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *na
if ((new_dataset->u.dataset.space_id = H5Scopy(space_id)) < 0)
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTCOPY, NULL, "failed to copy dataset's dataspace");

if (rv_hash_table_insert(RV_type_info_array_g[H5I_DATASET]->table, (char *)new_dataset,
(char *)new_dataset) == 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, NULL, "Failed to add dataset to type info array");

ret_value = (void *)new_dataset;

done:
Expand Down Expand Up @@ -342,6 +346,12 @@ RV_dataset_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name
size_t path_size = 0;
size_t path_len = 0;

RV_type_info *type_info = RV_type_info_array_g[H5I_DATASET];
rv_hash_table_iter_t iterator;
rv_hash_table_iterate(type_info->table, &iterator);
hid_t matching_dspace = H5I_INVALID_HID;
RV_object_t *other_dataset = H5I_INVALID_HID;

#ifdef RV_CONNECTOR_DEBUG
printf("-> Received dataset open call with following parameters:\n");
printf(" - loc_id object's URI: %s\n", parent->URI);
Expand Down Expand Up @@ -394,7 +404,23 @@ RV_dataset_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name
#endif

/* Set up a Dataspace for the opened Dataset */
if ((dataset->u.dataset.space_id = RV_parse_dataspace(response_buffer.buffer)) < 0)

/* If this is another view of an already-opened dataset, make them share the same dataspace
* so that changes to it (e.g. resizes) are visible to both views */
while (rv_hash_table_iter_has_more(&iterator)) {
other_dataset = (RV_object_t *)rv_hash_table_iter_next(&iterator);

if (!strcmp(other_dataset->URI, dataset->URI)) {
matching_dspace = other_dataset->u.dataset.space_id;
break;
}
}

if (matching_dspace != H5I_INVALID_HID) {
dataset->u.dataset.space_id = matching_dspace;
H5I_inc_ref(matching_dspace);
}
else if ((dataset->u.dataset.space_id = RV_parse_dataspace(response_buffer.buffer)) < 0)
FUNC_GOTO_ERROR(H5E_DATASPACE, H5E_CANTCONVERT, NULL,
"can't convert JSON to usable dataspace for dataset");

Expand Down Expand Up @@ -423,6 +449,9 @@ RV_dataset_open(void *obj, const H5VL_loc_params_t *loc_params, const char *name
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, NULL,
"can't parse dataset's creation properties from JSON representation");

if (rv_hash_table_insert(RV_type_info_array_g[H5I_DATASET]->table, (char *)dataset, (char *)dataset) == 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, NULL, "Failed to add dataset to type info array");

ret_value = (void *)dataset;

done:
Expand Down Expand Up @@ -1366,8 +1395,19 @@ RV_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **r
herr_t
RV_dataset_specific(void *obj, H5VL_dataset_specific_args_t *args, hid_t dxpl_id, void **req)
{
RV_object_t *dset = (RV_object_t *)obj;
herr_t ret_value = SUCCEED;
RV_object_t *dset = (RV_object_t *)obj;
herr_t ret_value = SUCCEED;
size_t host_header_len = 0;
char *host_header = NULL;
char *request_body = NULL;
char *request_body_shape = NULL;
char request_url[URL_MAX_LENGTH];
int url_len = 0;
hid_t dspace_id = H5I_INVALID_HID;
hid_t new_dspace_id = H5I_INVALID_HID;
hsize_t *old_extent = NULL;
hsize_t *maxdims = NULL;
upload_info uinfo;

#ifdef RV_CONNECTOR_DEBUG
printf("-> Received dataset-specific call with following parameters:\n");
Expand All @@ -1383,11 +1423,146 @@ RV_dataset_specific(void *obj, H5VL_dataset_specific_args_t *args, hid_t dxpl_id
switch (args->op_type) {
/* H5Dset_extent */
case H5VL_DATASET_SET_EXTENT:
size_t ndims = 0;
hsize_t *new_extent = NULL;

/* Check for write access */
if (!(dset->domain->u.file.intent & H5F_ACC_RDWR))
FUNC_GOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "no write intent on file");

FUNC_GOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "H5Dset_extent is unsupported");
if (!args->args.set_extent.size)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "given dimension array is NULL");

new_extent = args->args.set_extent.size;

/* Assemble curl request */
if ((url_len =
snprintf(request_url, URL_MAX_LENGTH, "%s/datasets/%s/shape", base_URL, dset->URI)) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_SYSERRSTR, FAIL, "snprintf error");

if (url_len >= URL_MAX_LENGTH)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_SYSERRSTR, FAIL,
"H5Dset_extent request URL size exceeded maximum URL size");

/* Setup the host header */
host_header_len = strlen(dset->domain->u.file.filepath_name) + strlen(host_string) + 1;
if (NULL == (host_header = (char *)RV_malloc(host_header_len)))
FUNC_GOTO_ERROR(H5E_SYM, H5E_CANTALLOC, FAIL, "can't allocate space for request Host header");

strcpy(host_header, host_string);

curl_headers =
curl_slist_append(curl_headers, strncat(host_header, dset->domain->u.file.filepath_name,
host_header_len - strlen(host_string) - 1));

/* Disable use of Expect: 100 Continue HTTP response */
curl_headers = curl_slist_append(curl_headers, "Expect:");

if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers))
FUNC_GOTO_ERROR(H5E_SYM, H5E_CANTSET, FAIL, "can't set cURL HTTP headers: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_URL, request_url))
FUNC_GOTO_ERROR(H5E_SYM, H5E_CANTSET, FAIL, "can't set cURL request URL: %s", curl_err_buf);

/* First, make a GET request for the dataset's shape to do some checks */
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPGET, 1))
FUNC_GOTO_ERROR(H5E_SYM, H5E_CANTSET, FAIL, "can't set up cURL to make HTTP GET request: %s",
curl_err_buf);

CURL_PERFORM(curl, H5E_DATASET, H5E_CANTGET, FAIL);

if ((dspace_id = RV_parse_dataspace(response_buffer.buffer)) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_PARSEERROR, FAIL, "failed to parse dataspace");

if ((ndims = H5Sget_simple_extent_ndims(dspace_id)) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "failed to get number of dataset dimensions");

if ((old_extent = calloc((size_t)ndims, sizeof(hssize_t))) == NULL)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL,
"failed to allocate memory for dataset shape");

if ((maxdims = calloc((size_t)ndims, sizeof(hssize_t))) == NULL)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL,
"failed to allocate memory for dataset max shape");

if (H5Sget_simple_extent_dims(dspace_id, old_extent, maxdims) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "failed to get dataset dimensions");

for (size_t i = 0; i < (size_t)ndims; i++)
if (new_extent[i] > maxdims[i])
FUNC_GOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL,
"new dataset dimensions exceed maximum dimensions");

/* HSDS limitation */
for (size_t i = 0; i < (size_t)ndims; i++)
if (new_extent[i] < old_extent[i])
FUNC_GOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"using H5Dset_extent to decrease dataset size is unsupported");

/* Construct JSON containing new dataset extent */
const char *fmt_string = "{"
"\"shape\": [%s]"
"}";

/* Compute space needed for request */
size_t request_body_shape_size = 0;

for (size_t i = 0; i < ndims; i++) {
/* N bytes needed to store an N digit number,
* floor(log10) + 1 of an N digit number is >= N,
* plus two bytes for space and comma characters in the list */
request_body_shape_size += floor(log10((double)new_extent[i])) + 1 + 2;
}

if ((request_body = RV_malloc(request_body_shape_size + strlen(fmt_string) + 1)) == NULL)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate memory for request body");

if ((request_body_shape = RV_malloc(request_body_shape_size)) == NULL)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL,
"can't allocate memory for request body shape");

request_body_shape[0] = '\0';
char *curr_internal_ptr = request_body_shape;

char dim_buffer[URL_MAX_LENGTH];

for (size_t i = 0; i < ndims; i++) {
int dim_len = snprintf(dim_buffer, URL_MAX_LENGTH, "%zu", new_extent[i]);

strcat(curr_internal_ptr, dim_buffer);
curr_internal_ptr += dim_len;

if (i != ndims - 1) {
strcat(curr_internal_ptr, ", ");
curr_internal_ptr += 2;
}
}

snprintf(request_body, request_body_shape_size + strlen(fmt_string) + 1, fmt_string,
request_body_shape);

uinfo.buffer = request_body;
uinfo.buffer_size = (size_t)strlen(request_body);
uinfo.bytes_sent = 0;

/* Make PUT request to change dataset extent */
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L))
FUNC_GOTO_ERROR(H5E_SYM, H5E_CANTSET, FAIL, "can't set up cURL to make HTTP PUT request: %s",
curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_READDATA, &uinfo))
FUNC_GOTO_ERROR(H5E_ATTR, H5E_CANTSET, FAIL, "can't set cURL PUT data: %s", curl_err_buf);

if (CURLE_OK !=
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)strlen(request_body)))
FUNC_GOTO_ERROR(H5E_ATTR, H5E_CANTSET, FAIL, "can't set cURL PUT data size: %s",
curl_err_buf);

CURL_PERFORM(curl, H5E_DATASET, H5E_CANTGET, FAIL);

/* Modify local dataspace to match version on server */
if (H5Sset_extent_simple(dset->u.dataset.space_id, ndims, new_extent, maxdims) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_DATASPACE, FAIL,
"unable to modify extent of local dataspace");

break;

/* H5Dflush */
Expand All @@ -1407,6 +1582,29 @@ RV_dataset_specific(void *obj, H5VL_dataset_specific_args_t *args, hid_t dxpl_id
done:
PRINT_ERROR_STACK;

/* Unset cURL UPLOAD option to ensure that future requests don't try to use PUT calls */
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_UPLOAD, 0))
FUNC_DONE_ERROR(H5E_ATTR, H5E_CANTSET, FAIL, "can't unset cURL PUT option: %s", curl_err_buf);

if (host_header)
RV_free(host_header);

if (curl_headers) {
curl_slist_free_all(curl_headers);
curl_headers = NULL;
}

if (dspace_id != H5I_INVALID_HID)
H5Sclose(dspace_id);

if ((ret_value < 0) && (new_dspace_id != H5I_INVALID_HID))
H5Sclose(new_dspace_id);

RV_free(old_extent);
RV_free(request_body);
RV_free(request_body_shape);
RV_free(maxdims);

return ret_value;
} /* end RV_dataset_specific() */

Expand Down Expand Up @@ -1458,6 +1656,9 @@ RV_dataset_close(void *dset, hid_t dxpl_id, void **req)
FUNC_DONE_ERROR(H5E_PLIST, H5E_CANTCLOSEOBJ, FAIL, "can't close DCPL");
} /* end if */

if (RV_type_info_array_g[H5I_DATASET])
rv_hash_table_remove(RV_type_info_array_g[H5I_DATASET]->table, (char *)_dset);

if (RV_file_close(_dset->domain, H5P_DEFAULT, NULL)) {
FUNC_DONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close file");
}
Expand Down
11 changes: 11 additions & 0 deletions src/rest_vol_datatype.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,10 @@ RV_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *n
if (RV_parse_response(response_buffer.buffer, NULL, new_datatype->URI, RV_copy_object_URI_callback) < 0)
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, NULL, "can't parse committed datatype's URI");

if (rv_hash_table_insert(RV_type_info_array_g[H5I_DATATYPE]->table, (char *)new_datatype,
(char *)new_datatype) == 0)
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTALLOC, NULL, "Failed to add datatype to type info array");

ret_value = (void *)new_datatype;

done:
Expand Down Expand Up @@ -419,6 +423,10 @@ RV_datatype_open(void *obj, const H5VL_loc_params_t *loc_params, const char *nam
if ((datatype->u.datatype.tcpl_id = H5Pcreate(H5P_DATATYPE_CREATE)) < 0)
FUNC_GOTO_ERROR(H5E_PLIST, H5E_CANTCREATE, NULL, "can't create TCPL for datatype");

if (rv_hash_table_insert(RV_type_info_array_g[H5I_DATATYPE]->table, (char *)datatype, (char *)datatype) ==
0)
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTALLOC, NULL, "Failed to add datatype to type info array");

ret_value = (void *)datatype;

done:
Expand Down Expand Up @@ -567,6 +575,9 @@ RV_datatype_close(void *dt, hid_t dxpl_id, void **req)
FUNC_DONE_ERROR(H5E_PLIST, H5E_CANTCLOSEOBJ, FAIL, "can't close TCPL");
} /* end if */

if (RV_type_info_array_g[H5I_DATATYPE])
rv_hash_table_remove(RV_type_info_array_g[H5I_DATATYPE]->table, (char *)_dtype);

if (RV_file_close(_dtype->domain, H5P_DEFAULT, NULL) < 0)
FUNC_DONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close file");

Expand Down
Loading

0 comments on commit a680f50

Please sign in to comment.