Skip to content

Commit

Permalink
convert-hf : Fix the encoding in the convert-hf-to-gguf-update.py (gg…
Browse files Browse the repository at this point in the history
  • Loading branch information
hamdoudhakem authored Jun 20, 2024
1 parent abd894a commit 17b291a
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions convert-hf-to-gguf-update.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,15 +214,15 @@ def get_vocab_base_pre(self, tokenizer) -> str:
"""

convert_py_pth = pathlib.Path("convert-hf-to-gguf.py")
convert_py = convert_py_pth.read_text()
convert_py = convert_py_pth.read_text(encoding="utf-8")
convert_py = re.sub(
r"(# Marker: Start get_vocab_base_pre)(.+?)( +# Marker: End get_vocab_base_pre)",
lambda m: m.group(1) + src_func + m.group(3),
convert_py,
flags=re.DOTALL | re.MULTILINE,
)

convert_py_pth.write_text(convert_py)
convert_py_pth.write_text(convert_py, encoding="utf-8")

logger.info("+++ convert-hf-to-gguf.py was updated")

Expand Down

0 comments on commit 17b291a

Please sign in to comment.