diff --git a/README.md b/README.md index d4cb04d..a87a0ca 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,9 @@ Golang binding for https://github.com/zejunwang1/easytokenizer ### Version +* version 0.2.1 + * Update some free function. + * version 0.2.0 * Fix `EncodeWithIds`, `WordPieceTokenize` API return error result. * Add Github Workflows. diff --git a/tokenizer/tokenizer.cc b/tokenizer/tokenizer.cc index caddab0..6736756 100644 --- a/tokenizer/tokenizer.cc +++ b/tokenizer/tokenizer.cc @@ -242,7 +242,10 @@ std::string BasicTokenizer::normalize(const uint8_t* str) const n = 0; } } - free(norm); + if (norm) { + free(norm); + norm = NULL; + } return result; } @@ -370,7 +373,10 @@ SizeT Tokenizer::NFD_codepoint_number(const uint8_t* str) const n = 0; } } - free(norm); + if (norm) { + free(norm); + norm = NULL; + } return c; } diff --git a/tokenizer/utf8proc.c b/tokenizer/utf8proc.c index 6752a55..af2f746 100644 --- a/tokenizer/utf8proc.c +++ b/tokenizer/utf8proc.c @@ -740,11 +740,13 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom( result = utf8proc_decompose_custom(str, strlen, buffer, result, options, custom_func, custom_data); if (result < 0) { free(buffer); + buffer = NULL; return result; } result = utf8proc_reencode(buffer, result, options); if (result < 0) { free(buffer); + buffer = NULL; return result; } {