Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Speed up import type (0.05 s to 0.02 s) by lazy loading requests #328

Merged
merged 6 commits into from
Oct 3, 2023
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pooch/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@
import shlex
import shutil

import requests
import requests.exceptions

from .hashes import hash_matches, file_hash
from .utils import (
Expand Down Expand Up @@ -757,6 +755,8 @@ def stream_download(url, fname, known_hash, downloader, pooch=None, retry_if_fai
will retry the download the specified number of times in case the failure
was due to a network error.
"""
import requests.exceptions # pylint: disable=C0415
leouieda marked this conversation as resolved.
Show resolved Hide resolved

# Ensure the parent directory exists in case the file is in a subdirectory.
# Otherwise, move will cause an error.
if not fname.parent.exists():
Expand Down
17 changes: 15 additions & 2 deletions pooch/downloaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@
import sys
import ftplib

import requests

from .utils import parse_url

try:
Expand Down Expand Up @@ -191,6 +189,9 @@ def __call__(self, url, output_file, pooch, check_only=False):
is available on the server. Otherwise, returns ``None``.

"""
# Lazy import requests to speed up import time
import requests # pylint: disable=C0415

if check_only:
response = requests.head(url, allow_redirects=True)
available = bool(response.status_code == 200)
Expand Down Expand Up @@ -648,6 +649,9 @@ def doi_to_url(doi):
The URL of the archive in the data repository.

"""
# Lazy import requests to speed up import time
import requests # pylint: disable=C0415

# Use doi.org to resolve the DOI to the repository website.
response = requests.get(f"https://doi.org/{doi}")
url = response.url
Expand Down Expand Up @@ -745,6 +749,9 @@ def download_url(self, file_name):
download_url : str
The HTTP URL that can be used to download the file.
"""
# Lazy import requests to speed up import time
import requests # pylint: disable=C0415

article_id = self.archive_url.split("/")[-1]
# With the ID, we can get a list of files and their download links
article = requests.get(f"https://zenodo.org/api/records/{article_id}").json()
Expand Down Expand Up @@ -803,6 +810,8 @@ def download_url(self, file_name):
download_url : str
The HTTP URL that can be used to download the file.
"""
# Lazy import requests to speed up import time
import requests # pylint: disable=C0415

# Use the figshare API to find the article ID from the DOI
article = requests.get(
Expand Down Expand Up @@ -846,6 +855,8 @@ def initialize(cls, doi, archive_url):
archive_url : str
The resolved URL for the DOI
"""
# Lazy import requests to speed up import time
import requests # pylint: disable=C0415

# Access the DOI as if this was a DataVerse instance
parsed = parse_url(archive_url)
Expand Down Expand Up @@ -875,6 +886,8 @@ def download_url(self, file_name):
download_url : str
The HTTP URL that can be used to download the file.
"""
# Lazy import requests to speed up import time
import requests # pylint: disable=C0415

# Access the DOI as if this was a DataVerse instance
parsed = parse_url(self.archive_url)
Expand Down