reset https client on multiprocessing

Author: lhoestq

src/huggingface_hub/hf_file_system.py

@@ -23,7 +23,7 @@
 from .errors import EntryNotFoundError, HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError
 from .file_download import hf_hub_url, http_get
 from .hf_api import HfApi, LastCommitInfo, RepoFile
-from .utils import HFValidationError, hf_raise_for_status, http_backoff, http_stream_backoff
+from .utils import HFValidationError, close_session, hf_raise_for_status, http_backoff, http_stream_backoff
 
 
 # Regex used to match special revisions with "/" in them (see #1710)
@@ -85,8 +85,14 @@ def __init__(cls, *args, **kwargs):
         # collecting instances when all other references are gone. To really
         # delete a FileSystem, the cache must be cleared.
         cls._cache = {}
+        cls._pid = os.getpid()
 
     def __call__(cls, *args, **kwargs):
+        if os.getpid() != cls._pid:
+            # We need to reset the session in case of multiprocessing.
+            # In particular we observed SSL errors and corrupted requests with multiprocessing "fork".
+            close_session()
+            cls._pid = os.getpid()
         skip = kwargs.pop("skip_instance_cache", False)
         fs_token = cls._tokenize(cls, threading.get_ident(), *args, **kwargs)
         fs_token_main_thread = cls._tokenize(cls, threading.main_thread().ident, *args, **kwargs)