Skip to content

Commit

Permalink
Deprecated use_auth_token (#1837)
Browse files Browse the repository at this point in the history 
* deprecated use_auth_token

* fix

* style

* import offline error

* style
  • Loading branch information
@IlyasMoutawwakil
IlyasMoutawwakil authored Jun 25, 2024
1 parent aad4b8b commit 60efc2a
Show file tree
Hide file tree
Showing 14 changed files with 369 additions and 107 deletions.
30 changes: 26 additions & 4 deletions optimum/configuration_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
import json
import os
import re
import warnings
from typing import Any, Dict, List, Tuple, Union

from huggingface_hub import HfFolder
from packaging import version
from transformers import PretrainedConfig
from transformers import __version__ as transformers_version_str
Expand Down Expand Up @@ -93,7 +93,19 @@ def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub:
repo_id = self._create_repo(repo_id, **kwargs)

use_auth_token = kwargs.get("use_auth_token", None)
token = HfFolder.get_token() if use_auth_token is True else use_auth_token
token = kwargs.get("token", None)

if use_auth_token is not None:
warnings.warn(
"The `use_auth_token` argument is deprecated and will be removed soon. Please use the `token` argument instead.",
FutureWarning,
)
if token is not None:
raise ValueError(
"You cannot use both `use_auth_token` and `token` arguments at the same time."
)
kwargs["token"] = use_auth_token
token = use_auth_token

files_timestamps = self._get_files_timestamps(save_directory)

Expand Down Expand Up @@ -197,6 +209,7 @@ def _get_config_dict(
resume_download = kwargs.pop("resume_download", False)
proxies = kwargs.pop("proxies", None)
use_auth_token = kwargs.pop("use_auth_token", None)
token = kwargs.pop("token", None)
local_files_only = kwargs.pop("local_files_only", False)
revision = kwargs.pop("revision", None)
trust_remote_code = kwargs.pop("trust_remote_code", None)
Expand All @@ -205,6 +218,15 @@ def _get_config_dict(
from_auto_class = kwargs.pop("_from_auto", False)
commit_hash = kwargs.pop("_commit_hash", None)

if use_auth_token is not None:
warnings.warn(
"The `use_auth_token` argument is deprecated and will be removed soon. Please use the `token` argument instead.",
FutureWarning,
)
if token is not None:
raise ValueError("You cannot use both `use_auth_token` and `token` arguments at the same time.")
token = use_auth_token

if trust_remote_code is True:
logger.warning(
"The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is"
Expand Down Expand Up @@ -255,7 +277,7 @@ def _get_config_dict(
proxies=proxies,
resume_download=resume_download,
local_files_only=local_files_only,
use_auth_token=use_auth_token,
token=token,
user_agent=user_agent,
)
else:
Expand All @@ -268,7 +290,7 @@ def _get_config_dict(
proxies=proxies,
resume_download=resume_download,
local_files_only=local_files_only,
use_auth_token=use_auth_token,
token=token,
user_agent=user_agent,
revision=revision,
subfolder=subfolder,
Expand Down
21 changes: 17 additions & 4 deletions optimum/exporters/onnx/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
"""Entry point to the optimum.exporters.onnx command line."""

import argparse
import warnings
from pathlib import Path

from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE
Expand Down Expand Up @@ -66,6 +67,7 @@ def main_export(
force_download: bool = False,
local_files_only: bool = False,
use_auth_token: Optional[Union[bool, str]] = None,
token: Optional[Union[bool, str]] = None,
for_ort: bool = False,
do_validation: bool = True,
model_kwargs: Optional[Dict[str, Any]] = None,
Expand Down Expand Up @@ -135,9 +137,11 @@ def main_export(
cached versions if they exist.
local_files_only (`Optional[bool]`, defaults to `False`):
Whether or not to only look at local files (i.e., do not try to download the model).
use_auth_token (`Optional[str]`, defaults to `None`):
use_auth_token (`Optional[Union[bool,str]]`, defaults to `None`):
Deprecated. Please use the `token` argument instead.
token (`Optional[Union[bool,str]]`, defaults to `None`):
The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
when running `transformers-cli login` (stored in `~/.huggingface`).
when running `huggingface-cli login` (stored in `huggingface_hub.constants.HF_TOKEN_PATH`).
model_kwargs (`Optional[Dict[str, Any]]`, defaults to `None`):
Experimental usage: keyword arguments to pass to the model during
the export. This argument should be used along the `custom_onnx_configs` argument
Expand Down Expand Up @@ -174,6 +178,15 @@ def main_export(
```
"""

if use_auth_token is not None:
warnings.warn(
"The `use_auth_token` argument is deprecated and will be removed soon. Please use the `token` argument instead.",
FutureWarning,
)
if token is not None:
raise ValueError("You cannot use both `use_auth_token` and `token` arguments at the same time.")
token = use_auth_token

if fp16:
if dtype is not None:
raise ValueError(
Expand Down Expand Up @@ -250,7 +263,7 @@ def main_export(
subfolder=subfolder,
revision=revision,
cache_dir=cache_dir,
use_auth_token=use_auth_token,
token=token,
local_files_only=local_files_only,
force_download=force_download,
trust_remote_code=trust_remote_code,
Expand Down Expand Up @@ -283,7 +296,7 @@ def main_export(
subfolder=subfolder,
revision=revision,
cache_dir=cache_dir,
use_auth_token=use_auth_token,
token=token,
local_files_only=local_files_only,
force_download=force_download,
trust_remote_code=trust_remote_code,
Expand Down
58 changes: 48 additions & 10 deletions optimum/exporters/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@
import inspect
import itertools
import os
import warnings
from functools import partial
from pathlib import Path
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union

import huggingface_hub
from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE
from huggingface_hub.errors import OfflineModeIsEnabled
from packaging import version
from requests.exceptions import ConnectionError as RequestsConnectionError
from transformers import AutoConfig, PretrainedConfig, is_tf_available, is_torch_available
Expand Down Expand Up @@ -1391,9 +1393,19 @@ def get_model_files(
model_name_or_path: Union[str, Path],
subfolder: str = "",
cache_dir: str = HUGGINGFACE_HUB_CACHE,
use_auth_token: Optional[str] = None,
use_auth_token: Optional[Union[bool, str]] = None,
token: Optional[Union[bool, str]] = None,
revision: Optional[str] = None,
):
if use_auth_token is not None:
warnings.warn(
"The `use_auth_token` argument is deprecated and will be removed soon. Please use the `token` argument instead.",
FutureWarning,
)
if token is not None:
raise ValueError("You cannot use both `use_auth_token` and `token` arguments at the same time.")
token = use_auth_token

request_exception = None
full_model_path = Path(model_name_or_path) / subfolder
if full_model_path.is_dir():
Expand All @@ -1409,12 +1421,12 @@ def get_model_files(
all_files = huggingface_hub.list_repo_files(
model_name_or_path,
repo_type="model",
token=use_auth_token,
token=token,
revision=revision,
)
if subfolder != "":
all_files = [file[len(subfolder) + 1 :] for file in all_files if file.startswith(subfolder)]
except (RequestsConnectionError, huggingface_hub.utils._http.OfflineModeIsEnabled) as e:
except (RequestsConnectionError, OfflineModeIsEnabled) as e:
request_exception = e
object_id = model_name_or_path.replace("/", "--")
full_model_path = Path(cache_dir, f"models--{object_id}")
Expand Down Expand Up @@ -1588,7 +1600,7 @@ def _infer_task_from_model_name_or_path(
)
try:
model_info = huggingface_hub.model_info(model_name_or_path, revision=revision)
except (RequestsConnectionError, huggingface_hub.utils._http.OfflineModeIsEnabled):
except (RequestsConnectionError, OfflineModeIsEnabled):
raise RuntimeError(
f"Hugging Face Hub is not reachable and we cannot infer the task from a cached model. Make sure you are not offline, or otherwise please specify the `task` (or `--task` in command-line) argument ({', '.join(TasksManager.get_all_tasks())})."
)
Expand Down Expand Up @@ -1706,7 +1718,8 @@ def infer_library_from_model(
revision: Optional[str] = None,
cache_dir: str = HUGGINGFACE_HUB_CACHE,
library_name: Optional[str] = None,
use_auth_token: Optional[str] = None,
use_auth_token: Optional[Union[bool, str]] = None,
token: Optional[Union[bool, str]] = None,
):
"""
Infers the library from the model repo.
Expand All @@ -1724,16 +1737,30 @@ def infer_library_from_model(
Path to a directory in which a downloaded pretrained model weights have been cached if the standard cache should not be used.
library_name (`Optional[str]`, *optional*):
The library name of the model. Can be any of "transformers", "timm", "diffusers", "sentence_transformers".
use_auth_token (`Optional[str]`, defaults to `None`):
The token to use as HTTP bearer authorization for remote files.
use_auth_token (`Optional[Union[bool,str]]`, defaults to `None`):
Deprecated. Please use the `token` argument instead.
token (`Optional[Union[bool,str]]`, defaults to `None`):
The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
when running `huggingface-cli login` (stored in `huggingface_hub.constants.HF_TOKEN_PATH`).
Returns:
`str`: The library name automatically detected from the model repo.
"""

if use_auth_token is not None:
warnings.warn(
"The `use_auth_token` argument is deprecated and will be removed soon. Please use the `token` argument instead.",
FutureWarning,
)
if token is not None:
raise ValueError("You cannot use both `use_auth_token` and `token` arguments at the same time.")
token = use_auth_token

if library_name is not None:
return library_name

all_files, _ = TasksManager.get_model_files(
model_name_or_path, subfolder, cache_dir, use_auth_token=use_auth_token
model_name_or_path, subfolder, cache_dir, token=token, revision=revision
)

if "model_index.json" in all_files:
Expand All @@ -1749,7 +1776,7 @@ def infer_library_from_model(
"subfolder": subfolder,
"revision": revision,
"cache_dir": cache_dir,
"use_auth_token": use_auth_token,
"token": token,
}
config_dict, kwargs = PretrainedConfig.get_config_dict(model_name_or_path, **kwargs)
model_config = PretrainedConfig.from_dict(config_dict, **kwargs)
Expand Down Expand Up @@ -1924,12 +1951,23 @@ def get_model_from_task(
elif library_name == "sentence_transformers":
cache_folder = model_kwargs.pop("cache_folder", None)
use_auth_token = model_kwargs.pop("use_auth_token", None)
token = model_kwargs.pop("token", None)
trust_remote_code = model_kwargs.pop("trust_remote_code", False)

if use_auth_token is not None:
warnings.warn(
"The `use_auth_token` argument is deprecated and will be removed soon. Please use the `token` argument instead.",
FutureWarning,
)
if token is not None:
raise ValueError("You cannot use both `use_auth_token` and `token` arguments at the same time.")
token = use_auth_token

model = model_class(
model_name_or_path,
device=device,
cache_folder=cache_folder,
use_auth_token=use_auth_token,
token=token,
trust_remote_code=trust_remote_code,
)
else:
Expand Down
Loading

0 comments on commit 60efc2a

Please sign in to comment.