Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add: use token_str from vocab #34

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

add: use token_str from vocab #34

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9f4a6b9
add: use token_str from vocab
juvi21 Aug 3, 2024
043712c
Update tekken.py
juvi21 Aug 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions src/mistral_common/tokens/tokenizers/tekken.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ def __init__(
)
self._vocab_size = vocab_size
self._path = _path
self._tokens_str = {token['rank']: token['token_str'] for token in vocab if token['token_str'] is not None}

special_tokens = list(self.SPECIAL_TOKENS)
assert len(special_tokens) == len(set(special_tokens)), f"Special tokens must be unique: {special_tokens}"
Expand Down Expand Up @@ -215,12 +216,17 @@ def _decode_all(self, tokens: List[int], special_token_policy: SpecialTokenPolic
decoded.extend(self._all_special_tokens[t] for t in group)
elif special_token_policy == SpecialTokenPolicy.IGNORE:
continue
# TODO: Could use "tokens_str" from vocab.json
# but need to handle null cases.

else:
decoded.append(self._model.decode([t - self.num_special_tokens for t in group]))
decoded.extend(self._decode_token(t) for t in group)
return decoded

def _decode_token(self, token: int) -> str:
adjusted_token = token - self.num_special_tokens
if adjusted_token in self._tokens_str:
return self._tokens_str[adjusted_token]
return self._model.decode([adjusted_token])

def is_byte(self, token_id: int) -> bool:
return 0 <= token_id - self.num_special_tokens < 256

Expand Down