diff --git a/hsds/basenode.py b/hsds/basenode.py index 28b31826..3525c5a6 100644 --- a/hsds/basenode.py +++ b/hsds/basenode.py @@ -33,7 +33,7 @@ from .util.k8sClient import getDnLabelSelector, getPodIps from . import hsds_logger as log -HSDS_VERSION = "0.8.0" +HSDS_VERSION = "0.8.1" def getVersion(): diff --git a/hsds/chunk_crawl.py b/hsds/chunk_crawl.py index e06b072d..96497e1c 100755 --- a/hsds/chunk_crawl.py +++ b/hsds/chunk_crawl.py @@ -27,11 +27,11 @@ from .util.httpUtil import isUnixDomainUrl from .util.idUtil import getDataNodeUrl, getNodeCount from .util.hdf5dtype import createDataType -from .util.dsetUtil import getFillValue, getSliceQueryParam +from .util.dsetUtil import getSliceQueryParam from .util.dsetUtil import getSelectionShape, getChunkLayout from .util.chunkUtil import getChunkCoverage, getDataCoverage from .util.chunkUtil import getChunkIdForPartition, getQueryDtype -from .util.arrayUtil import jsonToArray, getShapeDims +from .util.arrayUtil import jsonToArray, getShapeDims, getNumpyValue from .util.arrayUtil import getNumElements, arrayToBytes, bytesToArray from . import config from . import hsds_logger as log @@ -401,11 +401,18 @@ async def read_point_sel( params["action"] = "get" params["count"] = num_points - fill_value = getFillValue(dset_json) - np_arr_rsp = None dt = np_arr.dtype + fill_value = None + # initialize to fill_value if specified + if "creationProperties" in dset_json: + cprops = dset_json["creationProperties"] + if "fillValue" in cprops: + fill_value_prop = cprops["fillValue"] + encoding = cprops.get("fillValue_encoding") + fill_value = getNumpyValue(fill_value_prop, dt=dt, encoding=encoding) + def defaultArray(): # no data, return zero array if fill_value: diff --git a/hsds/chunk_sn.py b/hsds/chunk_sn.py index 5ef334db..73f0e5ed 100755 --- a/hsds/chunk_sn.py +++ b/hsds/chunk_sn.py @@ -31,7 +31,7 @@ from .util.domainUtil import getBucketForDomain from .util.hdf5dtype import getItemSize, createDataType from .util.dsetUtil import getSelectionList, isNullSpace, getDatasetLayoutClass -from .util.dsetUtil import getFillValue, isExtensible, getSelectionPagination +from .util.dsetUtil import isExtensible, getSelectionPagination from .util.dsetUtil import getSelectionShape, getDsetMaxDims, getChunkLayout from .util.dsetUtil import getDatasetCreationPropertyLayout from .util.chunkUtil import getNumChunks, getChunkIds, getChunkId @@ -40,7 +40,7 @@ from .util.chunkUtil import getQueryDtype, get_chunktable_dims from .util.arrayUtil import bytesArrayToList, jsonToArray, getShapeDims from .util.arrayUtil import getNumElements, arrayToBytes, bytesToArray -from .util.arrayUtil import squeezeArray +from .util.arrayUtil import squeezeArray, getNumpyValue from .util.authUtil import getUserPasswordFromRequest, validateUserPassword from .util.boolparser import BooleanParser from .servicenode_lib import getObjectJson, validateAction @@ -1372,10 +1372,19 @@ async def doReadSelection( log.error(msg) raise HTTPBadRequest(reason=msg) - arr = np.zeros(np_shape, dtype=dset_dtype, order="C") - fill_value = getFillValue(dset_json) - if fill_value is not None: + # initialize to fill_value if specified + fill_value = None + if "creationProperties" in dset_json: + cprops = dset_json["creationProperties"] + if "fillValue" in cprops: + fill_value_prop = cprops["fillValue"] + encoding = cprops.get("fillValue_encoding") + fill_value = getNumpyValue(fill_value_prop, dt=dset_dtype, encoding=encoding) + if fill_value: + arr = np.empty(np_shape, dtype=dset_dtype, order="C") arr[...] = fill_value + else: + arr = np.zeros(np_shape, dtype=dset_dtype, order="C") crawler = ChunkCrawler( app, diff --git a/hsds/datanode_lib.py b/hsds/datanode_lib.py index 7989d2cd..7a8e1e74 100644 --- a/hsds/datanode_lib.py +++ b/hsds/datanode_lib.py @@ -29,10 +29,10 @@ from .util.domainUtil import isValidDomain, getBucketForDomain from .util.attrUtil import getRequestCollectionName from .util.httpUtil import http_post -from .util.dsetUtil import getChunkLayout, getFilterOps, getFillValue +from .util.dsetUtil import getChunkLayout, getFilterOps from .util.dsetUtil import getChunkInitializer, getSliceQueryParam from .util.chunkUtil import getDatasetId, getChunkSelection, getChunkIndex -from .util.arrayUtil import arrayToBytes, bytesToArray, getShapeDims, jsonToArray +from .util.arrayUtil import arrayToBytes, bytesToArray, getShapeDims, jsonToArray, getNumpyValue from .util.hdf5dtype import createDataType, getItemSize from .util.rangegetUtil import ChunkLocation, chunkMunge @@ -1119,11 +1119,14 @@ async def get_chunk( if chunk_arr is None: # normal fill value based init or initializer failed - fill_value = getFillValue(dset_json) + fill_value = None + if "creationProperties" in dset_json: + cprops = dset_json["creationProperties"] + if "fillValue" in cprops: + fill_value_prop = cprops["fillValue"] + encoding = cprops.get("fillValue_encoding") + fill_value = getNumpyValue(fill_value_prop, dt=dt, encoding=encoding) if fill_value: - # need to convert list to tuples for numpy broadcast - if isinstance(fill_value, list): - fill_value = tuple(fill_value) chunk_arr = np.empty(dims, dtype=dt, order="C") chunk_arr[...] = fill_value else: diff --git a/hsds/dset_sn.py b/hsds/dset_sn.py index 09f2fe8e..e9e8729c 100755 --- a/hsds/dset_sn.py +++ b/hsds/dset_sn.py @@ -15,7 +15,6 @@ # import math -import numpy as np from json import JSONDecodeError from aiohttp.web_exceptions import HTTPBadRequest, HTTPNotFound, HTTPConflict @@ -23,7 +22,7 @@ from .util.httpUtil import jsonResponse from .util.idUtil import isValidUuid, getDataNodeUrl, createObjId, isSchema2Id from .util.dsetUtil import getPreviewQuery, getFilterItem -from .util.arrayUtil import getNumElements, getShapeDims +from .util.arrayUtil import getNumElements, getShapeDims, getNumpyValue from .util.chunkUtil import getChunkSize, guessChunk, expandChunk, shrinkChunk from .util.chunkUtil import getContiguousLayout from .util.authUtil import getUserPasswordFromRequest, aclCheck @@ -1031,26 +1030,29 @@ async def POST_Dataset(request): # validate fill value compatible with type dt = createDataType(datatype) fill_value = creationProperties["fillValue"] - is_nan = False - if dt.kind == "f": - if isinstance(fill_value, str) and fill_value == "nan": - is_nan = True - - if is_nan: - # use np.nan as fill value - # TBD: this needs to be fixed up for compound types - log.debug("converting 'nan' to np.nan for fillValue") - creationProperties["fillValue"] = np.nan - else: - if isinstance(fill_value, list): - fill_value = tuple(fill_value) - try: - np.asarray(fill_value, dtype=dt) - except (TypeError, ValueError): - msg = f"Fill value {fill_value} not compatible with " - msg += f"dataset type: {datatype}" + if "fillValue_encoding" in creationProperties: + fill_value_encoding = creationProperties["fillValue_encoding"] + + if fill_value_encoding not in ("None", "base64"): + msg = f"unexpected value for fill_value_encoding: {fill_value_encoding}" log.warn(msg) raise HTTPBadRequest(reason=msg) + else: + # should see a string in this case + if not isinstance(fill_value, str): + msg = f"unexpected fill value: {fill_value} " + msg += f"for encoding: {fill_value_encoding}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) + else: + fill_value_encoding = None + + try: + getNumpyValue(fill_value, dt=dt, encoding=fill_value_encoding) + except ValueError: + msg = f"invalid fill value: {fill_value}" + log.warn(msg) + raise HTTPBadRequest(reason=msg) if "filters" in creationProperties: # convert to standard representation diff --git a/hsds/util/arrayUtil.py b/hsds/util/arrayUtil.py index 0b8fd4bd..9bc8c1ca 100644 --- a/hsds/util/arrayUtil.py +++ b/hsds/util/arrayUtil.py @@ -11,18 +11,19 @@ ############################################################################## import math +import base64 +import binascii import numpy as np -MAX_VLEN_ELEMENT = 1000000 # restrict largest vlen element to one million - -""" -Convert list that may contain bytes type elements to list of string elements - -TBD: Need to deal with non-string byte data (hexencode?) -""" +MAX_VLEN_ELEMENT = 1_000_000 # restrict largest vlen element to one million def bytesArrayToList(data): + """ + Convert list that may contain bytes type elements to list of string elements + + TBD: Need to deal with non-string byte data (hexencode?) + """ if type(data) in (bytes, str): is_list = False elif isinstance(data, (np.ndarray, np.generic)): @@ -52,13 +53,11 @@ def bytesArrayToList(data): return out -""" -Convert a list to a tuple, recursively. -Example. [[1,2],[3,4]] -> ((1,2),(3,4)) -""" - - def toTuple(rank, data): + """ + Convert a list to a tuple, recursively. + Example. [[1,2],[3,4]] -> ((1,2),(3,4)) + """ if type(data) in (list, tuple): if rank > 0: return list(toTuple(rank - 1, x) for x in data) @@ -68,24 +67,20 @@ def toTuple(rank, data): return data -""" -Get size in bytes of a numpy array. -""" - - def getArraySize(arr): + """ + Get size in bytes of a numpy array. + """ nbytes = arr.dtype.itemsize for n in arr.shape: nbytes *= n return nbytes -""" -Helper - get num elements defined by a shape -""" - - def getNumElements(dims): + """ + Get num elements defined by a shape + """ num_elements = 0 if isinstance(dims, int): num_elements = dims @@ -98,13 +93,11 @@ def getNumElements(dims): return num_elements -""" -Get dims from a given shape json. Return [1,] for Scalar datasets, - None for null dataspaces -""" - - def getShapeDims(shape): + """ + Get dims from a given shape json. Return [1,] for Scalar datasets, + None for null dataspaces + """ dims = None if isinstance(shape, int): dims = [ @@ -138,13 +131,10 @@ def getShapeDims(shape): return dims -""" -Return numpy array from the given json array. -""" - - def jsonToArray(data_shape, data_dtype, data_json): - # utility function to initialize vlen array + """ + Return numpy array from the given json array. + """ def fillVlenArray(rank, data, arr, index): for i in range(len(data)): if rank > 1: @@ -199,12 +189,10 @@ def fillVlenArray(rank, data, arr, index): return arr -""" -Return True if the type contains variable length elements -""" - - def isVlen(dt): + """ + Return True if the type contains variable length elements + """ is_vlen = False if len(dt) > 1: names = dt.names @@ -218,12 +206,10 @@ def isVlen(dt): return is_vlen -""" -Get number of byte needed to given element as a bytestream -""" - - def getElementSize(e, dt): + """ + Get number of byte needed to given element as a bytestream + """ # print(f"getElementSize - e: {e} dt: {dt} metadata: {dt.metadata}") if len(dt) > 1: count = 0 @@ -267,12 +253,10 @@ def getElementSize(e, dt): return count -""" -Get number of bytes needed to store given numpy array as a bytestream -""" - - def getByteArraySize(arr): + """ + Get number of bytes needed to store given numpy array as a bytestream + """ if not isVlen(arr.dtype): return arr.itemsize * math.prod(arr.shape) nElements = math.prod(arr.shape) @@ -285,12 +269,10 @@ def getByteArraySize(arr): return count -""" -Copy to buffer at given offset -""" - - def copyBuffer(src, des, offset): + """ + Copy to buffer at given offset + """ # print(f"copyBuffer - src: {src} offset: {offset}") # TBD: just do: des[offset:] = src[:] ? for i in range(len(src)): @@ -300,12 +282,10 @@ def copyBuffer(src, des, offset): return offset + len(src) -""" -Copy element to bytearray -""" - - def copyElement(e, dt, buffer, offset): + """ + Copy element to bytearray + """ # print(f"copyElement - dt: {dt} offset: {offset}") if len(dt) > 1: for name in dt.names: @@ -388,12 +368,10 @@ def copyElement(e, dt, buffer, offset): return offset -""" -Get the count value from persisted vlen array -""" - - def getElementCount(buffer, offset): + """ + Get the count value from persisted vlen array + """ n = offset m = offset + 4 count_bytes = bytes(buffer[n:m]) @@ -412,12 +390,10 @@ def getElementCount(buffer, offset): return count -""" -Read element from bytearrray -""" - - def readElement(buffer, offset, arr, index, dt): + """ + Read element from bytearrray + """ # print(f"readElement, offset: {offset}, index: {index} dt: {dt}") if len(dt) > 1: @@ -473,12 +449,10 @@ def readElement(buffer, offset, arr, index, dt): return offset -""" -Return byte representation of numpy array -""" - - def arrayToBytes(arr): + """ + Return byte representation of numpy array + """ if not isVlen(arr.dtype): # can just return normal numpy bytestream return arr.tobytes() @@ -494,12 +468,10 @@ def arrayToBytes(arr): return bytes(buffer) -""" -Create numpy array based on byte representation -""" - - def bytesToArray(data, dt, shape): + """ + Create numpy array based on byte representation + """ # print(f"bytesToArray({len(data)}, {dt}, {shape}") nelements = getNumElements(shape) if not isVlen(dt): @@ -523,15 +495,46 @@ def bytesToArray(data, dt, shape): return arr -""" -Reduce dimensions by removing any 1-extent dimensions. -Just return input if no 1-extent dimensions +def getNumpyValue(value, dt=None, encoding=None): + """ + Return value as numpy type for given dtype and encoding + Encoding is expected to be one of None or "base64" + """ + # create a scalar numpy array + arr = np.zeros((), dtype=dt) + + if encoding and not isinstance(value, str): + msg = "Expected value to be string to use encoding" + raise ValueError(msg) -Note: only works with ndarrays (for now at least) -""" + if encoding == "base64": + try: + data = base64.decodebytes(value.encode("utf-8")) + except binascii.Error: + msg = "Unable to decode base64 string: {value}" + # log.warn(msg) + raise ValueError(msg) + arr = bytesToArray(data, dt, ()) + else: + if isinstance(value, list): + # convert to tuple + value = tuple(value) + elif dt.kind == "f" and isinstance(value, str) and value == "nan": + value = np.NaN + else: + # use as is + pass + arr = np.asarray(value, dtype=dt) + return arr[()] def squeezeArray(data): + """ + Reduce dimensions by removing any 1-extent dimensions. + Just return input if no 1-extent dimensions + + Note: only works with ndarrays (for now at least) + """ if not isinstance(data, np.ndarray): raise TypeError("expected ndarray") if len(data.shape) <= 1: @@ -612,13 +615,11 @@ def __next__(self): return tuple(ret_index) -# compare two numpy arrays. -# return true if the same (exclusive of null vs. empty array) -# false otherwise -# TBD: this is slow for multi-megabyte vlen arrays, needs to be optimized - - def ndarray_compare(arr1, arr2): + # compare two numpy arrays. + # return true if the same (exclusive of null vs. empty array) + # false otherwise + # TBD: this is slow for multi-megabyte vlen arrays, needs to be optimized if not isinstance(arr1, np.ndarray) and not isinstance(arr2, np.ndarray): if not isinstance(arr1, np.void) and not isinstance(arr2, np.void): return arr1 == arr2 diff --git a/hsds/util/dsetUtil.py b/hsds/util/dsetUtil.py index 667a8731..9df8c0fe 100644 --- a/hsds/util/dsetUtil.py +++ b/hsds/util/dsetUtil.py @@ -837,20 +837,6 @@ def getPreviewQuery(dims): return select -def getFillValue(dset_json): - """ - Return fill value if defined, otherwise return None - """ - fill_value = None - if "creationProperties" in dset_json: - cprops = dset_json["creationProperties"] - if "fillValue" in cprops: - fill_value = cprops["fillValue"] - if isinstance(fill_value, list): - fill_value = tuple(fill_value) - return fill_value - - def isExtensible(dims, maxdims): """ Determine if the dataset can be extended diff --git a/hsds/util/hdf5dtype.py b/hsds/util/hdf5dtype.py index af0150aa..67119491 100644 --- a/hsds/util/hdf5dtype.py +++ b/hsds/util/hdf5dtype.py @@ -551,7 +551,7 @@ def getNumpyTypename(hdf5TypeName, typeClass=None): def createBaseDataType(typeItem): dtRet = None - if type(typeItem) == str: + if isinstance(typeItem, str): # should be one of the predefined types dtName = getNumpyTypename(typeItem) dtRet = np.dtype(dtName) @@ -717,7 +717,7 @@ def createDataType(typeItem): dtRet = np.dtype(dtName) return dtRet # return predefined type - if type(typeItem) != dict: + if not isinstance(typeItem, dict): raise TypeError("invalid type") if "class" not in typeItem: @@ -735,7 +735,7 @@ def createDataType(typeItem): subtypes = [] for field in fields: - if type(field) != dict: + if not isinstance(field, dict): raise TypeError("Expected dictionary type for field") if "name" not in field: raise KeyError("'name' missing from field") diff --git a/hsds/util/storUtil.py b/hsds/util/storUtil.py index 7c832945..704c2980 100644 --- a/hsds/util/storUtil.py +++ b/hsds/util/storUtil.py @@ -18,13 +18,10 @@ import json import zlib import numcodecs as codecs -from aiohttp.web_exceptions import HTTPNotFound, HTTPInternalServerError -from aiohttp.client_exceptions import ClientError -from asyncio import CancelledError +from aiohttp.web_exceptions import HTTPInternalServerError from .. import hsds_logger as log -from .httpUtil import http_get from .s3Client import S3Client try: @@ -195,45 +192,6 @@ def getURIFromKey(app, bucket=None, key=None): return uri -async def rangegetProxy(app, bucket=None, key=None, offset=0, length=0): - """fetch bytes from rangeget proxy""" - if "rangeget_url" in app: - req = app["rangeget_url"] + "/" - else: - rangeget_port = config.get("rangeget_port") - if "is_docker" in app: - rangeget_host = "rangeget" - else: - rangeget_host = "127.0.0.1" - req = f"http://{rangeget_host}:{rangeget_port}/" - log.debug(f"rangeGetProxy: {req}") - params = {} - params["bucket"] = bucket - params["key"] = key - params["offset"] = offset - params["length"] = length - - try: - data = await http_get(app, req, params=params) - log.debug(f"rangeget request: {req}, read {len(data)} bytes") - except HTTPNotFound: - # external HDF5 file, should exist - log.warn(f"range get request not found for params: {params}") - raise - except ClientError as ce: - log.error(f"Error for rangeget({req}): {ce} ") - raise HTTPInternalServerError() - except CancelledError as cle: - log.warn(f"CancelledError for rangeget request({req}): {cle}") - return None - - if len(data) != length: - msg = f"expected {length} bytes for rangeget {bucket}{key}, " - msg += f"but got: {len(data)}" - log.warn(msg) - return data - - async def getStorJSONObj(app, key, bucket=None): """Get object identified by key and read as JSON""" diff --git a/setup.py b/setup.py index 913e8c3f..2d2a748a 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ setup( name="hsds", - version="0.8.0", + version="0.8.1", description="HDF REST API", url="http://github.com/HDFGroup/hsds", author="John Readey", diff --git a/tests/integ/dataset_test.py b/tests/integ/dataset_test.py index 5e2f623e..ff50d138 100755 --- a/tests/integ/dataset_test.py +++ b/tests/integ/dataset_test.py @@ -1370,7 +1370,65 @@ def get_payload(dset_type, fillValue=None): self.assertTrue("creationProperties" in rspJson) creationProps = rspJson["creationProperties"] self.assertTrue("fillValue" in creationProps) - self.assertTrue(np.isnan(creationProps["fillValue"])) + fillValue = creationProps["fillValue"] + self.assertEqual(fillValue, "nan") + + def testNaNFillValueBase64Encoded(self): + # test Dataset with simple type and fill value that is incompatible with the type + domain = self.base_domain + "/testNaNFillValueBase64Encoded.h5" + helper.setupDomain(domain) + print("testNaNFillValueBase64Encoded", domain) + headers = helper.getRequestHeaders(domain=domain) + + # get domain + req = helper.getEndpoint() + "/" + rsp = self.session.get(req, headers=headers) + rspJson = json.loads(rsp.text) + self.assertTrue("root" in rspJson) + root_uuid = rspJson["root"] + + def get_payload(dset_type, fillValue=None, encoding=None): + payload = {"type": dset_type, "shape": 10} + if fillValue is not None: + cprops = {"fillValue": fillValue} + if encoding: + cprops["fillValue_encoding"] = encoding + payload["creationProperties"] = cprops + return payload + + # create the dataset + req = self.endpoint + "/datasets" + + payload = get_payload("H5T_IEEE_F32LE", fillValue="AADAfw==", encoding="base64") + req = self.endpoint + "/datasets" + rsp = self.session.post(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) # Dataset created + rspJson = json.loads(rsp.text) + dset_uuid = rspJson["id"] + + # link new dataset + req = self.endpoint + "/groups/" + root_uuid + "/links/dset1" + payload = {"id": dset_uuid} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) + + # verify creationProperties + req = helper.getEndpoint() + "/datasets/" + dset_uuid + rsp = self.session.get(req, headers=headers) + self.assertEqual(rsp.status_code, 200) + rspJson = json.loads(rsp.text) + self.assertTrue("creationProperties" in rspJson) + creationProps = rspJson["creationProperties"] + self.assertTrue("fillValue" in creationProps) + self.assertEqual(creationProps["fillValue"], "AADAfw==") + self.assertTrue("fillValue_encoding" in creationProps) + self.assertEqual(creationProps["fillValue_encoding"], "base64") + + # link new dataset + req = self.endpoint + "/groups/" + root_uuid + "/links/dset2" + payload = {"id": dset_uuid} + rsp = self.session.put(req, data=json.dumps(payload), headers=headers) + self.assertEqual(rsp.status_code, 201) def testAutoChunk1dDataset(self): # test Dataset where chunk layout is set automatically diff --git a/tests/integ/helper.py b/tests/integ/helper.py index 9c89ea0a..d5d5412d 100644 --- a/tests/integ/helper.py +++ b/tests/integ/helper.py @@ -53,7 +53,7 @@ def getRangeGetEndpoint(): def validateId(id): """Return true if the parameter looks like a UUID""" - if type(id) != str: + if not isinstance(id, str): # should be a string return False if len(id) != 38: diff --git a/tests/integ/vlen_test.py b/tests/integ/vlen_test.py index 75886445..074fa26f 100755 --- a/tests/integ/vlen_test.py +++ b/tests/integ/vlen_test.py @@ -78,7 +78,7 @@ def testPutVLenInt(self): # write values to dataset data = [ - [1,], + [1, ], [1, 2], [1, 2, 3], [1, 2, 3, 4], diff --git a/tests/perf/smallobj/small_obj_test.py b/tests/perf/smallobj/small_obj_test.py index 6f86e2f3..f1c87e63 100644 --- a/tests/perf/smallobj/small_obj_test.py +++ b/tests/perf/smallobj/small_obj_test.py @@ -157,11 +157,11 @@ async def store(group_name): for key, val in things.items(): logging.debug(f"{key}: {val}") - if type(val) == dict: + if isinstance(val, dict): logging.debug("dict") val_grp_id = await create_group(group_id, key) logging.debug(f"got val_grp_id: {val_grp_id}") - elif type(val) == ThingItem: + elif isinstance(val, ThingItem): logging.info(f"ThingItem - create_group_attributes name for group: {group_id} ") val_grp_id = await create_group(group_id, key) await create_attribute(val_grp_id, "name", val.name) @@ -177,7 +177,7 @@ async def store_items(grp_names): session = ClientSession(loop=loop, connector=TCPConnector(limit=max_tcp_connections)) globals["client"] = session xs = stream.iterate(grp_names) | pipe.map(store, ordered=False, task_limit=task_limit) - await(xs) + await xs await session.close() diff --git a/tests/perf/stream/helper.py b/tests/perf/stream/helper.py index 8eabebbe..15cece55 100644 --- a/tests/perf/stream/helper.py +++ b/tests/perf/stream/helper.py @@ -53,7 +53,7 @@ def getRangeGetEndpoint(): def validateId(id): """Return true if the parameter looks like a UUID""" - if type(id) != str: + if not isinstance(id, str): # should be a string return False if len(id) != 38: diff --git a/tests/unit/array_util_test.py b/tests/unit/array_util_test.py index 56bc4351..669c1b1c 100644 --- a/tests/unit/array_util_test.py +++ b/tests/unit/array_util_test.py @@ -25,7 +25,8 @@ bytesToArray, getByteArraySize, IndexIterator, - ndarray_compare + ndarray_compare, + getNumpyValue ) from hsds.util.hdf5dtype import special_dtype from hsds.util.hdf5dtype import check_dtype @@ -673,6 +674,71 @@ def testIndexIterator(self): cnt += 1 self.assertEqual(cnt, 20) + def testGetNumpyValue(self): + # test int conversion + dt = np.dtype("