Skip to content

Commit

Permalink
fix hf_backbone ut
Browse files Browse the repository at this point in the history
  • Loading branch information
lvyufeng committed Oct 23, 2024
1 parent 3ddbe30 commit 828c4ba
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 3 deletions.
8 changes: 6 additions & 2 deletions mindnlp/core/serialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -1408,11 +1408,15 @@ def convert(info: dict[str, Any]):
assert end - begin == math.prod(shape) * np.dtype(numpy_dtype).itemsize
buf = byte_buf[begin:end]
array = np.frombuffer(buf, dtype=numpy_dtype).reshape(shape)

if array.dtype == bfloat16 and not SUPPORT_BF16:
logger.warning_once("MindSpore do not support bfloat16 dtype, we will automaticlly convert to float16")
array = array.astype(np.float16)

return Tensor(array)

if info['dtype'] == 'I64':
array = array.astype(numpy_dtype)
out = Tensor(array)
return out

with open(filename, "rb") as fp:
header_size, = struct.unpack('<Q', fp.read(8))
Expand Down
1 change: 0 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,6 @@ def run(self):
'addict',
'ml_dtypes',
'pyctcdecode',
'jieba',
'pytest==7.2.0',
'pillow>=10.0.0'
],
Expand Down

0 comments on commit 828c4ba

Please sign in to comment.