update some free function

sunhailin-Leo · Nov 24, 2023 · 5e49044 · 5e49044
1 parent c2fc870
commit 5e49044
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -16,6 +16,9 @@ Golang binding for https://github.com/zejunwang1/easytokenizer
 
 ### Version
 
+* version 0.2.1
+  * Update some free function. 
+
 * version 0.2.0
   * Fix `EncodeWithIds`, `WordPieceTokenize` API return error result.
   * Add Github Workflows.

diff --git a/tokenizer/tokenizer.cc b/tokenizer/tokenizer.cc
@@ -242,7 +242,10 @@ std::string BasicTokenizer::normalize(const uint8_t* str) const
       n = 0;
     }
   }
-  free(norm);
+  if (norm) {
+    free(norm);
+    norm = NULL;
+  }
   return result;
 }
 
@@ -370,7 +373,10 @@ SizeT Tokenizer::NFD_codepoint_number(const uint8_t* str) const
       n = 0;
     }
   }
-  free(norm);
+  if (norm) {
+    free(norm);
+    norm = NULL;
+  }
   return c;
 }
 

diff --git a/tokenizer/utf8proc.c b/tokenizer/utf8proc.c
@@ -740,11 +740,13 @@ UTF8PROC_DLLEXPORT utf8proc_ssize_t utf8proc_map_custom(
   result = utf8proc_decompose_custom(str, strlen, buffer, result, options, custom_func, custom_data);
   if (result < 0) {
     free(buffer);
+    buffer = NULL;
     return result;
   }
   result = utf8proc_reencode(buffer, result, options);
   if (result < 0) {
     free(buffer);
+    buffer = NULL;
     return result;
   }
   {