Skip to content

Commit

Permalink
switch to default value of 'warn' instead of 'None'
Browse files Browse the repository at this point in the history
  • Loading branch information
Sagi Polaczek committed Oct 13, 2024
1 parent 33401ed commit 467e54c
Showing 1 changed file with 4 additions and 7 deletions.
11 changes: 4 additions & 7 deletions fuse/data/tokenizers/modular_tokenizer/op.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def __init__(
validate_ends_with_eos: Optional[bool] = True,
eos: Optional[str] = "<EOS>",
verbose: Optional[bool] = False,
on_unknown_default_value: Optional[str] = None,
on_unknown_default_value: str = "warn",
**kwargs: Any,
) -> None:
"""
Expand Down Expand Up @@ -62,10 +62,7 @@ def __init__(

self._validate_ends_with_eos = validate_ends_with_eos
self._eos = eos
if on_unknown_default_value is not None:
self._on_unknown_default_value = on_unknown_default_value
else:
self._on_unknown_default_value = "warn"
self._on_unknown_default_value = on_unknown_default_value

if on_unknown_default_value not in ["warn", "raise"]:
raise ValueError(f"Doesn't support {on_unknown_default_value=}!")
Expand Down Expand Up @@ -272,7 +269,7 @@ def __call__(
)

if on_unknown is None:
# Use tokenizer instance defautl value
# Use tokenizer instance default value
on_unknown = self._on_unknown_default_value

if isinstance(data, str):
Expand Down Expand Up @@ -522,7 +519,7 @@ def from_pretrained(
identifier: str,
pad_token: str = "<PAD>",
max_size: Optional[int] = None,
on_unknown_default_value: Optional[str] = None,
on_unknown_default_value: str = "warn",
force_download: bool = False,
resume_download: Optional[bool] = None,
proxies: Optional[Dict] = None,
Expand Down

0 comments on commit 467e54c

Please sign in to comment.