+# Copyright (c) 2018 The Pooch Developers.
+# Distributed under the terms of the BSD 3-Clause License.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# This code is part of the Fatiando a Terra project (https://www.fatiando.org)
+#
+# pylint: disable=missing-docstring,import-outside-toplevel,import-self
+#
+# Import functions/classes to make the API
+from.coreimportPooch,create,retrieve
+from.utilsimportos_cache,check_version,get_logger
+from.hashesimportfile_hash,make_registry
+from.downloadersimport(
+ HTTPDownloader,
+ FTPDownloader,
+ SFTPDownloader,
+ DOIDownloader,
+)
+from.processorsimportUnzip,Untar,Decompress
+
+# This file is generated automatically by setuptools_scm
+from.import_version
+
+
+# Add a "v" to the version number
+__version__=f"v{_version.version}"
+
+
+
+[docs]
+deftest(doctest=True,verbose=True,coverage=False):
+"""
+ Run the test suite.
+
+ Uses `py.test <http://pytest.org/>`__ to discover and run the tests.
+
+ Parameters
+ ----------
+
+ doctest : bool
+ If ``True``, will run the doctests as well (code examples that start
+ with a ``>>>`` in the docs).
+ verbose : bool
+ If ``True``, will print extra information during the test run.
+ coverage : bool
+ If ``True``, will run test coverage analysis on the code as well.
+ Requires ``pytest-cov``.
+
+ Raises
+ ------
+
+ AssertionError
+ If pytest returns a non-zero error code indicating that some tests have
+ failed.
+
+ """
+ importpytest
+
+ package=__name__
+ args=[]
+ ifverbose:
+ args.append("-vv")
+ ifcoverage:
+ args.append(f"--cov={package}")
+ args.append("--cov-report=term-missing")
+ ifdoctest:
+ args.append("--doctest-modules")
+ args.append("--pyargs")
+ args.append(package)
+ status=pytest.main(args)
+ assertstatus==0,"Some tests have failed."
+# Copyright (c) 2018 The Pooch Developers.
+# Distributed under the terms of the BSD 3-Clause License.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# This code is part of the Fatiando a Terra project (https://www.fatiando.org)
+#
+"""
+The main Pooch class and a factory function for it.
+"""
+importos
+importtime
+importcontextlib
+frompathlibimportPath
+importshlex
+importshutil
+
+
+from.hashesimporthash_matches,file_hash
+from.utilsimport(
+ check_version,
+ get_logger,
+ make_local_storage,
+ cache_location,
+ temporary_file,
+ os_cache,
+ unique_file_name,
+)
+from.downloadersimportDOIDownloader,choose_downloader,doi_to_repository
+
+
+
+[docs]
+defretrieve(
+ url,
+ known_hash,
+ fname=None,
+ path=None,
+ processor=None,
+ downloader=None,
+ progressbar=False,
+):
+"""
+ Download and cache a single file locally.
+
+ Uses HTTP or FTP by default, depending on the protocol in the given *url*.
+ Other download methods can be controlled through the *downloader* argument
+ (see below).
+
+ The file will be downloaded to a temporary location first and its hash will
+ be compared to the given *known_hash*. This is done to ensure that the
+ download happened correctly and securely. If the hash doesn't match, the
+ file will be deleted and an exception will be raised.
+
+ If the file already exists locally, its hash will be compared to
+ *known_hash*. If they are not the same, this is interpreted as the file
+ needing to be updated and it will be downloaded again.
+
+ You can bypass these checks by passing ``known_hash=None``. If this is
+ done, the SHA256 hash of the downloaded file will be logged to the screen.
+ It is highly recommended that you copy and paste this hash as *known_hash*
+ so that future downloads are guaranteed to be the exact same file. This is
+ crucial for reproducible computations.
+
+ If the file exists in the given *path* with the given *fname* and the hash
+ matches, it will not be downloaded and the absolute path to the file will
+ be returned.
+
+ .. note::
+
+ This function is meant for downloading single files. If you need to
+ manage the download and caching of several files, with versioning, use
+ :func:`pooch.create` and :class:`pooch.Pooch` instead.
+
+ Parameters
+ ----------
+ url : str
+ The URL to the file that is to be downloaded. Ideally, the URL should
+ end in a file name.
+ known_hash : str or None
+ A known hash (checksum) of the file. Will be used to verify the
+ download or check if an existing file needs to be updated. By default,
+ will assume it's a SHA256 hash. To specify a different hashing method,
+ prepend the hash with ``algorithm:``, for example
+ ``md5:pw9co2iun29juoh`` or ``sha1:092odwhi2ujdp2du2od2odh2wod2``. If
+ None, will NOT check the hash of the downloaded file or check if an
+ existing file needs to be updated.
+ fname : str or None
+ The name that will be used to save the file. Should NOT include the
+ full path, just the file name (it will be appended to *path*). If
+ None, will create a unique file name using a combination of the last
+ part of the URL (assuming it's the file name) and the MD5 hash of the
+ URL. For example, ``81whdo2d2e928yd1wi22-data-file.csv``. This ensures
+ that files from different URLs never overwrite each other, even if they
+ have the same name.
+ path : str or PathLike or None
+ The location of the cache folder on disk. This is where the file will
+ be saved. If None, will save to a ``pooch`` folder in the default cache
+ location for your operating system (see :func:`pooch.os_cache`).
+ processor : None or callable
+ If not None, then a function (or callable object) that will be called
+ before returning the full path and after the file has been downloaded
+ (if required). See :ref:`processors` for details.
+ downloader : None or callable
+ If not None, then a function (or callable object) that will be called
+ to download a given URL to a provided local file name. See
+ :ref:`downloaders` for details.
+ progressbar : bool or an arbitrary progress bar object
+ If True, will print a progress bar of the download to standard error
+ (stderr). Requires `tqdm <https://github.com/tqdm/tqdm>`__ to be
+ installed. Alternatively, an arbitrary progress bar object can be
+ passed. See :ref:`custom-progressbar` for details.
+
+ Returns
+ -------
+ full_path : str
+ The absolute path (including the file name) of the file in the local
+ storage.
+
+ Examples
+ --------
+
+ Download one of the data files from the Pooch repository on GitHub:
+
+ >>> import os
+ >>> from pooch import __version__, check_version, retrieve
+ >>> # Make a URL for the version of pooch we have installed
+ >>> url = "https://github.com/fatiando/pooch/raw/{}/data/tiny-data.txt"
+ >>> url = url.format(check_version(__version__, fallback="main"))
+ >>> # Download the file and save it locally. Will check the MD5 checksum of
+ >>> # the downloaded file against the given value to make sure it's the
+ >>> # right file. You can use other hashes by specifying different
+ >>> # algorithm names (sha256, sha1, etc).
+ >>> fname = retrieve(
+ ... url, known_hash="md5:70e2afd3fd7e336ae478b1e740a5f08e",
+ ... )
+ >>> with open(fname) as f:
+ ... print(f.read().strip())
+ # A tiny data file for test purposes only
+ 1 2 3 4 5 6
+ >>> # Running again won't trigger a download and only return the path to
+ >>> # the existing file.
+ >>> fname2 = retrieve(
+ ... url, known_hash="md5:70e2afd3fd7e336ae478b1e740a5f08e",
+ ... )
+ >>> print(fname2 == fname)
+ True
+ >>> os.remove(fname)
+
+ Files that are compressed with gzip, xz/lzma, or bzip2 can be automatically
+ decompressed by passing using the :class:`pooch.Decompress` processor:
+
+ >>> from pooch import Decompress
+ >>> # URLs to a gzip compressed version of the data file.
+ >>> url = ("https://github.com/fatiando/pooch/raw/{}/"
+ ... + "pooch/tests/data/tiny-data.txt.gz")
+ >>> url = url.format(check_version(__version__, fallback="main"))
+ >>> # By default, you would have to decompress the file yourself
+ >>> fname = retrieve(
+ ... url,
+ ... known_hash="md5:8812ba10b6c7778014fdae81b03f9def",
+ ... )
+ >>> print(os.path.splitext(fname)[1])
+ .gz
+ >>> # Use the processor to decompress after download automatically and
+ >>> # return the path to the decompressed file instead.
+ >>> fname2 = retrieve(
+ ... url,
+ ... known_hash="md5:8812ba10b6c7778014fdae81b03f9def",
+ ... processor=Decompress(),
+ ... )
+ >>> print(fname2 == fname)
+ False
+ >>> with open(fname2) as f:
+ ... print(f.read().strip())
+ # A tiny data file for test purposes only
+ 1 2 3 4 5 6
+ >>> os.remove(fname)
+ >>> os.remove(fname2)
+
+ When downloading archives (zip or tar), it can be useful to unpack them
+ after download to avoid having to do that yourself. Use the processors
+ :class:`pooch.Unzip` or :class:`pooch.Untar` to do this automatically:
+
+ >>> from pooch import Unzip
+ >>> # URLs to a zip archive with a single data file.
+ >>> url = ("https://github.com/fatiando/pooch/raw/{}/"
+ ... + "pooch/tests/data/tiny-data.zip")
+ >>> url = url.format(check_version(__version__, fallback="main"))
+ >>> # By default, you would get the path to the archive
+ >>> fname = retrieve(
+ ... url,
+ ... known_hash="md5:e9592cb46cf3514a1079051f8a148148",
+ ... )
+ >>> print(os.path.splitext(fname)[1])
+ .zip
+ >>> os.remove(fname)
+ >>> # Using the processor, the archive will be unzipped and a list with the
+ >>> # path to every file will be returned instead of a single path.
+ >>> fnames = retrieve(
+ ... url,
+ ... known_hash="md5:e9592cb46cf3514a1079051f8a148148",
+ ... processor=Unzip(),
+ ... )
+ >>> # There was only a single file in our archive.
+ >>> print(len(fnames))
+ 1
+ >>> with open(fnames[0]) as f:
+ ... print(f.read().strip())
+ # A tiny data file for test purposes only
+ 1 2 3 4 5 6
+ >>> for f in fnames:
+ ... os.remove(f)
+
+
+ """
+ ifpathisNone:
+ path=os_cache("pooch")
+ iffnameisNone:
+ fname=unique_file_name(url)
+ # Make the path absolute.
+ path=cache_location(path,env=None,version=None)
+
+ full_path=path.resolve()/fname
+ action,verb=download_action(full_path,known_hash)
+
+ ifactionin("download","update"):
+ # We need to write data, so create the local data directory if it
+ # doesn't already exist.
+ make_local_storage(path)
+
+ get_logger().info(
+ "%s data from '%s' to file '%s'.",
+ verb,
+ url,
+ str(full_path),
+ )
+
+ ifdownloaderisNone:
+ downloader=choose_downloader(url,progressbar=progressbar)
+
+ stream_download(url,full_path,known_hash,downloader,pooch=None)
+
+ ifknown_hashisNone:
+ get_logger().info(
+ "SHA256 hash of downloaded file: %s\n"
+ "Use this value as the 'known_hash' argument of 'pooch.retrieve'"
+ " to ensure that the file hasn't changed if it is downloaded again"
+ " in the future.",
+ file_hash(str(full_path)),
+ )
+
+ ifprocessorisnotNone:
+ returnprocessor(str(full_path),action,None)
+
+ returnstr(full_path)
+
+
+
+
+[docs]
+defcreate(
+ path,
+ base_url,
+ version=None,
+ version_dev="master",
+ env=None,
+ registry=None,
+ urls=None,
+ retry_if_failed=0,
+ allow_updates=True,
+):
+"""
+ Create a :class:`~pooch.Pooch` with sensible defaults to fetch data files.
+
+ If a version string is given, the Pooch will be versioned, meaning that the
+ local storage folder and the base URL depend on the project version. This
+ is necessary if your users have multiple versions of your library installed
+ (using virtual environments) and you updated the data files between
+ versions. Otherwise, every time a user switches environments would trigger
+ a re-download of the data. The version string will be appended to the local
+ storage path (for example, ``~/.mypooch/cache/v0.1``) and inserted into the
+ base URL (for example,
+ ``https://github.com/fatiando/pooch/raw/v0.1/data``). If the version string
+ contains ``+XX.XXXXX``, it will be interpreted as a development version.
+
+ Does **not** create the local data storage folder. The folder will only be
+ created the first time a download is attempted with
+ :meth:`pooch.Pooch.fetch`. This makes it safe to use this function at the
+ module level (so it's executed on ``import`` and the resulting
+ :class:`~pooch.Pooch` is a global variable).
+
+ Parameters
+ ----------
+ path : str, PathLike, list or tuple
+ The path to the local data storage folder. If this is a list or tuple,
+ we'll join the parts with the appropriate separator. The *version* will
+ be appended to the end of this path. Use :func:`pooch.os_cache` for a
+ sensible default.
+ base_url : str
+ Base URL for the remote data source. All requests will be made relative
+ to this URL. The string should have a ``{version}`` formatting mark in
+ it. We will call ``.format(version=version)`` on this string. If the
+ URL does not end in a ``'/'``, a trailing ``'/'`` will be added
+ automatically.
+ version : str or None
+ The version string for your project. Should be PEP440 compatible. If
+ None is given, will not attempt to format *base_url* and no subfolder
+ will be appended to *path*.
+ version_dev : str
+ The name used for the development version of a project. If your data is
+ hosted on Github (and *base_url* is a Github raw link), then
+ ``"master"`` is a good choice (default). Ignored if *version* is None.
+ env : str or None
+ An environment variable that can be used to overwrite *path*. This
+ allows users to control where they want the data to be stored. We'll
+ append *version* to the end of this value as well.
+ registry : dict or None
+ A record of the files that are managed by this Pooch. Keys should be
+ the file names and the values should be their hashes. Only files
+ in the registry can be fetched from the local storage. Files in
+ subdirectories of *path* **must use Unix-style separators** (``'/'``)
+ even on Windows.
+ urls : dict or None
+ Custom URLs for downloading individual files in the registry. A
+ dictionary with the file names as keys and the custom URLs as values.
+ Not all files in *registry* need an entry in *urls*. If a file has an
+ entry in *urls*, the *base_url* will be ignored when downloading it in
+ favor of ``urls[fname]``.
+ retry_if_failed : int
+ Retry a file download the specified number of times if it fails because
+ of a bad connection or a hash mismatch. By default, downloads are only
+ attempted once (``retry_if_failed=0``). Initially, will wait for 1s
+ between retries and then increase the wait time by 1s with each retry
+ until a maximum of 10s.
+ allow_updates : bool or str
+ Whether existing files in local storage that have a hash mismatch with
+ the registry are allowed to update from the remote URL. If a string is
+ passed, we will assume it's the name of an environment variable that
+ will be checked for the true/false value. If ``False``, any mismatch
+ with hashes in the registry will result in an error. Defaults to
+ ``True``.
+
+ Returns
+ -------
+ pooch : :class:`~pooch.Pooch`
+ The :class:`~pooch.Pooch` initialized with the given arguments.
+
+ Examples
+ --------
+
+ Create a :class:`~pooch.Pooch` for a release (v0.1):
+
+ >>> pup = create(path="myproject",
+ ... base_url="http://some.link.com/{version}/",
+ ... version="v0.1",
+ ... registry={"data.txt": "9081wo2eb2gc0u..."})
+ >>> print(pup.path.parts) # The path is a pathlib.Path
+ ('myproject', 'v0.1')
+ >>> # The local folder is only created when a dataset is first downloaded
+ >>> print(pup.path.exists())
+ False
+ >>> print(pup.base_url)
+ http://some.link.com/v0.1/
+ >>> print(pup.registry)
+ {'data.txt': '9081wo2eb2gc0u...'}
+ >>> print(pup.registry_files)
+ ['data.txt']
+
+ If this is a development version (12 commits ahead of v0.1), then the
+ ``version_dev`` will be used (defaults to ``"master"``):
+
+ >>> pup = create(path="myproject",
+ ... base_url="http://some.link.com/{version}/",
+ ... version="v0.1+12.do9iwd")
+ >>> print(pup.path.parts)
+ ('myproject', 'master')
+ >>> print(pup.base_url)
+ http://some.link.com/master/
+
+ Versioning is optional (but highly encouraged):
+
+ >>> pup = create(path="myproject",
+ ... base_url="http://some.link.com/",
+ ... registry={"data.txt": "9081wo2eb2gc0u..."})
+ >>> print(pup.path.parts) # The path is a pathlib.Path
+ ('myproject',)
+ >>> print(pup.base_url)
+ http://some.link.com/
+
+ To place the storage folder at a subdirectory, pass in a list and we'll
+ join the path for you using the appropriate separator for your operating
+ system:
+
+ >>> pup = create(path=["myproject", "cache", "data"],
+ ... base_url="http://some.link.com/{version}/",
+ ... version="v0.1")
+ >>> print(pup.path.parts)
+ ('myproject', 'cache', 'data', 'v0.1')
+
+ The user can overwrite the storage path by setting an environment variable:
+
+ >>> # The variable is not set so we'll use *path*
+ >>> pup = create(path=["myproject", "not_from_env"],
+ ... base_url="http://some.link.com/{version}/",
+ ... version="v0.1",
+ ... env="MYPROJECT_DATA_DIR")
+ >>> print(pup.path.parts)
+ ('myproject', 'not_from_env', 'v0.1')
+ >>> # Set the environment variable and try again
+ >>> import os
+ >>> os.environ["MYPROJECT_DATA_DIR"] = os.path.join("myproject", "env")
+ >>> pup = create(path=["myproject", "not_env"],
+ ... base_url="http://some.link.com/{version}/",
+ ... version="v0.1",
+ ... env="MYPROJECT_DATA_DIR")
+ >>> print(pup.path.parts)
+ ('myproject', 'env', 'v0.1')
+
+ """
+ ifversionisnotNone:
+ version=check_version(version,fallback=version_dev)
+ base_url=base_url.format(version=version)
+ # Don't create the cache folder here! This function is usually called in
+ # the module context (at import time), so touching the file system is not
+ # recommended. It could cause crashes when multiple processes/threads try
+ # to import at the same time (which would try to create the folder several
+ # times at once).
+ path=cache_location(path,env,version)
+ ifisinstance(allow_updates,str):
+ allow_updates=os.environ.get(allow_updates,"true").lower()!="false"
+ # add trailing "/"
+ base_url=base_url.rstrip("/")+"/"
+ pup=Pooch(
+ path=path,
+ base_url=base_url,
+ registry=registry,
+ urls=urls,
+ retry_if_failed=retry_if_failed,
+ allow_updates=allow_updates,
+ )
+ returnpup
+
+
+
+
+[docs]
+classPooch:
+"""
+ Manager for a local data storage that can fetch from a remote source.
+
+ Avoid creating ``Pooch`` instances directly. Use :func:`pooch.create`
+ instead.
+
+ Parameters
+ ----------
+ path : str
+ The path to the local data storage folder. The path must exist in the
+ file system.
+ base_url : str
+ Base URL for the remote data source. All requests will be made relative
+ to this URL.
+ registry : dict or None
+ A record of the files that are managed by this good boy. Keys should be
+ the file names and the values should be their hashes. Only files
+ in the registry can be fetched from the local storage. Files in
+ subdirectories of *path* **must use Unix-style separators** (``'/'``)
+ even on Windows.
+ urls : dict or None
+ Custom URLs for downloading individual files in the registry. A
+ dictionary with the file names as keys and the custom URLs as values.
+ Not all files in *registry* need an entry in *urls*. If a file has an
+ entry in *urls*, the *base_url* will be ignored when downloading it in
+ favor of ``urls[fname]``.
+ retry_if_failed : int
+ Retry a file download the specified number of times if it fails because
+ of a bad connection or a hash mismatch. By default, downloads are only
+ attempted once (``retry_if_failed=0``). Initially, will wait for 1s
+ between retries and then increase the wait time by 1s with each retry
+ until a maximum of 10s.
+ allow_updates : bool
+ Whether existing files in local storage that have a hash mismatch with
+ the registry are allowed to update from the remote URL. If ``False``,
+ any mismatch with hashes in the registry will result in an error.
+ Defaults to ``True``.
+
+ """
+
+ def__init__(
+ self,
+ path,
+ base_url,
+ registry=None,
+ urls=None,
+ retry_if_failed=0,
+ allow_updates=True,
+ ):
+ self.path=path
+ self.base_url=base_url
+ ifregistryisNone:
+ registry={}
+ self.registry=registry
+ ifurlsisNone:
+ urls={}
+ self.urls=dict(urls)
+ self.retry_if_failed=retry_if_failed
+ self.allow_updates=allow_updates
+
+ @property
+ defabspath(self):
+ "Absolute path to the local storage"
+ returnPath(os.path.abspath(os.path.expanduser(str(self.path))))
+
+ @property
+ defregistry_files(self):
+ "List of file names on the registry"
+ returnlist(self.registry)
+
+
+[docs]
+ deffetch(self,fname,processor=None,downloader=None,progressbar=False):
+"""
+ Get the absolute path to a file in the local storage.
+
+ If it's not in the local storage, it will be downloaded. If the hash of
+ the file in local storage doesn't match the one in the registry, will
+ download a new copy of the file. This is considered a sign that the
+ file was updated in the remote storage. If the hash of the downloaded
+ file still doesn't match the one in the registry, will raise an
+ exception to warn of possible file corruption.
+
+ Post-processing actions sometimes need to be taken on downloaded files
+ (unzipping, conversion to a more efficient format, etc). If these
+ actions are time or memory consuming, it would be best to do this only
+ once right after the file is downloaded. Use the *processor* argument
+ to specify a function that is executed after the download to perform
+ these actions. See :ref:`processors` for details.
+
+ Custom file downloaders can be provided through the *downloader*
+ argument. By default, Pooch will determine the download protocol from
+ the URL in the registry. If the server for a given file requires
+ authentication (username and password), use a downloader that support
+ these features. Downloaders can also be used to print custom messages
+ (like a progress bar), etc. See :ref:`downloaders` for details.
+
+ Parameters
+ ----------
+ fname : str
+ The file name (relative to the *base_url* of the remote data
+ storage) to fetch from the local storage.
+ processor : None or callable
+ If not None, then a function (or callable object) that will be
+ called before returning the full path and after the file has been
+ downloaded. See :ref:`processors` for details.
+ downloader : None or callable
+ If not None, then a function (or callable object) that will be
+ called to download a given URL to a provided local file name. See
+ :ref:`downloaders` for details.
+ progressbar : bool or an arbitrary progress bar object
+ If True, will print a progress bar of the download to standard
+ error (stderr). Requires `tqdm <https://github.com/tqdm/tqdm>`__ to
+ be installed. Alternatively, an arbitrary progress bar object can
+ be passed. See :ref:`custom-progressbar` for details.
+
+ Returns
+ -------
+ full_path : str
+ The absolute path (including the file name) of the file in the
+ local storage.
+
+ """
+ self._assert_file_in_registry(fname)
+
+ url=self.get_url(fname)
+ full_path=self.abspath/fname
+ known_hash=self.registry[fname]
+ action,verb=download_action(full_path,known_hash)
+
+ ifaction=="update"andnotself.allow_updates:
+ raiseValueError(
+ f"{fname} needs to update {full_path} but updates are disallowed."
+ )
+
+ ifactionin("download","update"):
+ # We need to write data, so create the local data directory if it
+ # doesn't already exist.
+ make_local_storage(str(self.abspath))
+
+ get_logger().info(
+ "%s file '%s' from '%s' to '%s'.",
+ verb,
+ fname,
+ url,
+ str(self.abspath),
+ )
+
+ ifdownloaderisNone:
+ downloader=choose_downloader(url,progressbar=progressbar)
+
+ stream_download(
+ url,
+ full_path,
+ known_hash,
+ downloader,
+ pooch=self,
+ retry_if_failed=self.retry_if_failed,
+ )
+
+ ifprocessorisnotNone:
+ returnprocessor(str(full_path),action,self)
+
+ returnstr(full_path)
+
+
+ def_assert_file_in_registry(self,fname):
+"""
+ Check if a file is in the registry and raise :class:`ValueError` if
+ it's not.
+ """
+ iffnamenotinself.registry:
+ raiseValueError(f"File '{fname}' is not in the registry.")
+
+
+[docs]
+ defget_url(self,fname):
+"""
+ Get the full URL to download a file in the registry.
+
+ Parameters
+ ----------
+ fname : str
+ The file name (relative to the *base_url* of the remote data
+ storage) to fetch from the local storage.
+
+ """
+ self._assert_file_in_registry(fname)
+ returnself.urls.get(fname,"".join([self.base_url,fname]))
+
+
+
+[docs]
+ defload_registry(self,fname):
+"""
+ Load entries from a file and add them to the registry.
+
+ Use this if you are managing many files.
+
+ Each line of the file should have file name and its hash separated by
+ a space. Hash can specify checksum algorithm using "alg:hash" format.
+ In case no algorithm is provided, SHA256 is used by default.
+ Only one file per line is allowed. Custom download URLs for individual
+ files can be specified as a third element on the line. Line comments
+ can be added and must be prepended with ``#``.
+
+ Parameters
+ ----------
+ fname : str | fileobj
+ Path (or open file object) to the registry file.
+
+ """
+ withcontextlib.ExitStack()asstack:
+ ifhasattr(fname,"read"):
+ # It's a file object
+ fin=fname
+ else:
+ # It's a file path
+ fin=stack.enter_context(open(fname,encoding="utf-8"))
+
+ forlinenum,lineinenumerate(fin):
+ ifisinstance(line,bytes):
+ line=line.decode("utf-8")
+
+ line=line.strip()
+ # skip line comments
+ ifline.startswith("#"):
+ continue
+
+ elements=shlex.split(line)
+ ifnotlen(elements)in[0,2,3]:
+ raiseOSError(
+ f"Invalid entry in Pooch registry file '{fname}': "
+ f"expected 2 or 3 elements in line {linenum+1} but got "
+ f"{len(elements)}. Offending entry: '{line}'"
+ )
+ ifelements:
+ file_name=elements[0]
+ file_checksum=elements[1]
+ iflen(elements)==3:
+ file_url=elements[2]
+ self.urls[file_name]=file_url
+ self.registry[file_name]=file_checksum.lower()
+
+
+
+[docs]
+ defload_registry_from_doi(self):
+"""
+ Populate the registry using the data repository API
+
+ Fill the registry with all the files available in the data repository,
+ along with their hashes. It will make a request to the data repository
+ API to retrieve this information. No file is downloaded during this
+ process.
+
+ .. important::
+
+ This method is intended to be used only when the ``base_url`` is
+ a DOI.
+ """
+
+ # Ensure that this is indeed a DOI-based pooch
+ downloader=choose_downloader(self.base_url)
+ ifnotisinstance(downloader,DOIDownloader):
+ raiseValueError(
+ f"Invalid base_url '{self.base_url}': "
+ +"Pooch.load_registry_from_doi is only implemented for DOIs"
+ )
+
+ # Create a repository instance
+ doi=self.base_url.replace("doi:","")
+ repository=doi_to_repository(doi)
+
+ # Call registry population for this repository
+ returnrepository.populate_registry(self)
+
+
+
+[docs]
+ defis_available(self,fname,downloader=None):
+"""
+ Check availability of a remote file without downloading it.
+
+ Use this method when working with large files to check if they are
+ available for download.
+
+ Parameters
+ ----------
+ fname : str
+ The file name (relative to the *base_url* of the remote data
+ storage).
+ downloader : None or callable
+ If not None, then a function (or callable object) that will be
+ called to check the availability of the file on the server. See
+ :ref:`downloaders` for details.
+
+ Returns
+ -------
+ status : bool
+ True if the file is available for download. False otherwise.
+
+ """
+ self._assert_file_in_registry(fname)
+ url=self.get_url(fname)
+ ifdownloaderisNone:
+ downloader=choose_downloader(url)
+ try:
+ available=downloader(url,None,self,check_only=True)
+ exceptTypeErroraserror:
+ error_msg=(
+ f"Downloader '{str(downloader)}' does not support availability checks."
+ )
+ raiseNotImplementedError(error_msg)fromerror
+ returnavailable
+
+
+
+
+defdownload_action(path,known_hash):
+"""
+ Determine the action that is needed to get the file on disk.
+
+ Parameters
+ ----------
+ path : PathLike
+ The path to the file on disk.
+ known_hash : str
+ A known hash (checksum) of the file. Will be used to verify the
+ download or check if an existing file needs to be updated. By default,
+ will assume it's a SHA256 hash. To specify a different hashing method,
+ prepend the hash with ``algorithm:``, for example
+ ``md5:pw9co2iun29juoh`` or ``sha1:092odwhi2ujdp2du2od2odh2wod2``.
+
+ Returns
+ -------
+ action, verb : str
+ The action that must be taken and the English verb (infinitive form of
+ *action*) used in the log:
+ * ``'download'``: File does not exist locally and must be downloaded.
+ * ``'update'``: File exists locally but needs to be updated.
+ * ``'fetch'``: File exists locally and only need to inform its path.
+
+
+ """
+ ifnotpath.exists():
+ action="download"
+ verb="Downloading"
+ elifnothash_matches(str(path),known_hash):
+ action="update"
+ verb="Updating"
+ else:
+ action="fetch"
+ verb="Fetching"
+ returnaction,verb
+
+
+defstream_download(url,fname,known_hash,downloader,pooch=None,retry_if_failed=0):
+"""
+ Stream the file and check that its hash matches the known one.
+
+ The file is first downloaded to a temporary file name in the cache folder.
+ It will be moved to the desired file name only if the hash matches the
+ known hash. Otherwise, the temporary file is deleted.
+
+ If the download fails for either a bad connection or a hash mismatch, we
+ will retry the download the specified number of times in case the failure
+ was due to a network error.
+ """
+ # Lazy import requests to speed up import time
+ importrequests.exceptions# pylint: disable=C0415
+
+ # Ensure the parent directory exists in case the file is in a subdirectory.
+ # Otherwise, move will cause an error.
+ ifnotfname.parent.exists():
+ os.makedirs(str(fname.parent))
+ download_attempts=1+retry_if_failed
+ max_wait=10
+ foriinrange(download_attempts):
+ try:
+ # Stream the file to a temporary so that we can safely check its
+ # hash before overwriting the original.
+ withtemporary_file(path=str(fname.parent))astmp:
+ downloader(url,tmp,pooch)
+ hash_matches(tmp,known_hash,strict=True,source=str(fname.name))
+ shutil.move(tmp,str(fname))
+ break
+ except(ValueError,requests.exceptions.RequestException):
+ ifi==download_attempts-1:
+ raise
+ retries_left=download_attempts-(i+1)
+ get_logger().info(
+ "Failed to download '%s'. "
+ "Will attempt the download again %d more time%s.",
+ str(fname.name),
+ retries_left,
+ "s"ifretries_left>1else"",
+ )
+ time.sleep(min(i+1,max_wait))
+
+# Copyright (c) 2018 The Pooch Developers.
+# Distributed under the terms of the BSD 3-Clause License.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# This code is part of the Fatiando a Terra project (https://www.fatiando.org)
+#
+"""
+The classes that actually handle the downloads.
+"""
+importos
+importsys
+importftplib
+
+importwarnings
+
+from.utilsimportparse_url
+
+try:
+ fromtqdmimporttqdm
+exceptImportError:
+ tqdm=None
+
+try:
+ importparamiko
+exceptImportError:
+ paramiko=None
+
+
+defchoose_downloader(url,progressbar=False):
+"""
+ Choose the appropriate downloader for the given URL based on the protocol.
+
+ Parameters
+ ----------
+ url : str
+ A URL (including protocol).
+ progressbar : bool or an arbitrary progress bar object
+ If True, will print a progress bar of the download to standard error
+ (stderr). Requires `tqdm <https://github.com/tqdm/tqdm>`__ to be
+ installed. Alternatively, an arbitrary progress bar object can be
+ passed. See :ref:`custom-progressbar` for details.
+
+ Returns
+ -------
+ downloader
+ A downloader class, like :class:`pooch.HTTPDownloader`,
+ :class:`pooch.FTPDownloader`, or :class: `pooch.SFTPDownloader`.
+
+ Examples
+ --------
+
+ >>> downloader = choose_downloader("http://something.com")
+ >>> print(downloader.__class__.__name__)
+ HTTPDownloader
+ >>> downloader = choose_downloader("https://something.com")
+ >>> print(downloader.__class__.__name__)
+ HTTPDownloader
+ >>> downloader = choose_downloader("ftp://something.com")
+ >>> print(downloader.__class__.__name__)
+ FTPDownloader
+ >>> downloader = choose_downloader("doi:DOI/filename.csv")
+ >>> print(downloader.__class__.__name__)
+ DOIDownloader
+
+ """
+ known_downloaders={
+ "ftp":FTPDownloader,
+ "https":HTTPDownloader,
+ "http":HTTPDownloader,
+ "sftp":SFTPDownloader,
+ "doi":DOIDownloader,
+ }
+
+ parsed_url=parse_url(url)
+ ifparsed_url["protocol"]notinknown_downloaders:
+ raiseValueError(
+ f"Unrecognized URL protocol '{parsed_url['protocol']}' in '{url}'. "
+ f"Must be one of {known_downloaders.keys()}."
+ )
+ downloader=known_downloaders[parsed_url["protocol"]](progressbar=progressbar)
+ returndownloader
+
+
+
+[docs]
+classHTTPDownloader:# pylint: disable=too-few-public-methods
+"""
+ Download manager for fetching files over HTTP/HTTPS.
+
+ When called, downloads the given file URL into the specified local file.
+ Uses the :mod:`requests` library to manage downloads.
+
+ Use with :meth:`pooch.Pooch.fetch` or :func:`pooch.retrieve` to customize
+ the download of files (for example, to use authentication or print a
+ progress bar).
+
+ Parameters
+ ----------
+ progressbar : bool or an arbitrary progress bar object
+ If True, will print a progress bar of the download to standard error
+ (stderr). Requires `tqdm <https://github.com/tqdm/tqdm>`__ to be
+ installed. Alternatively, an arbitrary progress bar object can be
+ passed. See :ref:`custom-progressbar` for details.
+ chunk_size : int
+ Files are streamed *chunk_size* bytes at a time instead of loading
+ everything into memory at one. Usually doesn't need to be changed.
+ **kwargs
+ All keyword arguments given when creating an instance of this class
+ will be passed to :func:`requests.get`.
+
+ Examples
+ --------
+
+ Download one of the data files from the Pooch repository:
+
+ >>> import os
+ >>> from pooch import __version__, check_version
+ >>> url = "https://github.com/fatiando/pooch/raw/{}/data/tiny-data.txt"
+ >>> url = url.format(check_version(__version__, fallback="main"))
+ >>> downloader = HTTPDownloader()
+ >>> # Not using with Pooch.fetch so no need to pass an instance of Pooch
+ >>> downloader(url=url, output_file="tiny-data.txt", pooch=None)
+ >>> os.path.exists("tiny-data.txt")
+ True
+ >>> with open("tiny-data.txt") as f:
+ ... print(f.read().strip())
+ # A tiny data file for test purposes only
+ 1 2 3 4 5 6
+ >>> os.remove("tiny-data.txt")
+
+ Authentication can be handled by passing a user name and password to
+ :func:`requests.get`. All arguments provided when creating an instance of
+ the class are forwarded to :func:`requests.get`. We'll use
+ ``auth=(username, password)`` to use basic HTTPS authentication. The
+ https://httpbin.org website allows us to make a fake a login request using
+ whatever username and password we provide to it:
+
+ >>> user = "doggo"
+ >>> password = "goodboy"
+ >>> # httpbin will ask for the user and password we provide in the URL
+ >>> url = f"https://httpbin.org/basic-auth/{user}/{password}"
+ >>> # Trying without the login credentials causes an error
+ >>> downloader = HTTPDownloader()
+ >>> try:
+ ... downloader(url=url, output_file="tiny-data.txt", pooch=None)
+ ... except Exception:
+ ... print("There was an error!")
+ There was an error!
+ >>> # Pass in the credentials to HTTPDownloader
+ >>> downloader = HTTPDownloader(auth=(user, password))
+ >>> downloader(url=url, output_file="tiny-data.txt", pooch=None)
+ >>> with open("tiny-data.txt") as f:
+ ... for line in f:
+ ... print(line.rstrip())
+ {
+ "authenticated": true,
+ "user": "doggo"
+ }
+ >>> os.remove("tiny-data.txt")
+
+ """
+
+ def__init__(self,progressbar=False,chunk_size=1024,**kwargs):
+ self.kwargs=kwargs
+ self.progressbar=progressbar
+ self.chunk_size=chunk_size
+ ifself.progressbarisTrueandtqdmisNone:
+ raiseValueError("Missing package 'tqdm' required for progress bars.")
+
+
+[docs]
+ def__call__(
+ self,url,output_file,pooch,check_only=False
+ ):# pylint: disable=R0914
+"""
+ Download the given URL over HTTP to the given output file.
+
+ Uses :func:`requests.get`.
+
+ Parameters
+ ----------
+ url : str
+ The URL to the file you want to download.
+ output_file : str or file-like object
+ Path (and file name) to which the file will be downloaded.
+ pooch : :class:`~pooch.Pooch`
+ The instance of :class:`~pooch.Pooch` that is calling this method.
+ check_only : bool
+ If True, will only check if a file exists on the server and
+ **without downloading the file**. Will return ``True`` if the file
+ exists and ``False`` otherwise.
+
+ Returns
+ -------
+ availability : bool or None
+ If ``check_only==True``, returns a boolean indicating if the file
+ is available on the server. Otherwise, returns ``None``.
+
+ """
+ # Lazy import requests to speed up import time
+ importrequests# pylint: disable=C0415
+
+ ifcheck_only:
+ timeout=self.kwargs.get("timeout",5)
+ response=requests.head(url,timeout=timeout,allow_redirects=True)
+ available=bool(response.status_code==200)
+ returnavailable
+
+ kwargs=self.kwargs.copy()
+ timeout=kwargs.pop("timeout",5)
+ kwargs.setdefault("stream",True)
+ ispath=nothasattr(output_file,"write")
+ ifispath:
+ # pylint: disable=consider-using-with
+ output_file=open(output_file,"w+b")
+ # pylint: enable=consider-using-with
+ try:
+ response=requests.get(url,timeout=timeout,**kwargs)
+ response.raise_for_status()
+ content=response.iter_content(chunk_size=self.chunk_size)
+ total=int(response.headers.get("content-length",0))
+ ifself.progressbarisTrue:
+ # Need to use ascii characters on Windows because there isn't
+ # always full unicode support
+ # (see https://github.com/tqdm/tqdm/issues/454)
+ use_ascii=bool(sys.platform=="win32")
+ progress=tqdm(
+ total=total,
+ ncols=79,
+ ascii=use_ascii,
+ unit="B",
+ unit_scale=True,
+ leave=True,
+ )
+ elifself.progressbar:
+ progress=self.progressbar
+ progress.total=total
+ forchunkincontent:
+ ifchunk:
+ output_file.write(chunk)
+ output_file.flush()
+ ifself.progressbar:
+ # Use the chunk size here because chunk may be much
+ # larger if the data are decompressed by requests after
+ # reading (happens with text files).
+ progress.update(self.chunk_size)
+ # Make sure the progress bar gets filled even if the actual number
+ # is chunks is smaller than expected. This happens when streaming
+ # text files that are compressed by the server when sending (gzip).
+ # Binary files don't experience this.
+ ifself.progressbar:
+ progress.reset()
+ progress.update(total)
+ progress.close()
+ finally:
+ ifispath:
+ output_file.close()
+ returnNone
+
+
+
+
+
+[docs]
+classFTPDownloader:# pylint: disable=too-few-public-methods
+"""
+ Download manager for fetching files over FTP.
+
+ When called, downloads the given file URL into the specified local file.
+ Uses the :mod:`ftplib` module to manage downloads.
+
+ Use with :meth:`pooch.Pooch.fetch` or :func:`pooch.retrieve` to customize
+ the download of files (for example, to use authentication or print a
+ progress bar).
+
+ Parameters
+ ----------
+ port : int
+ Port used for the FTP connection.
+ username : str
+ User name used to login to the server. Only needed if the server
+ requires authentication (i.e., no anonymous FTP).
+ password : str
+ Password used to login to the server. Only needed if the server
+ requires authentication (i.e., no anonymous FTP). Use the empty string
+ to indicate no password is required.
+ account : str
+ Some servers also require an "account" name for authentication.
+ timeout : int
+ Timeout in seconds for ftp socket operations, use None to mean no
+ timeout.
+ progressbar : bool
+ If True, will print a progress bar of the download to standard error
+ (stderr). Requires `tqdm <https://github.com/tqdm/tqdm>`__ to be
+ installed. **Custom progress bars are not yet supported.**
+ chunk_size : int
+ Files are streamed *chunk_size* bytes at a time instead of loading
+ everything into memory at one. Usually doesn't need to be changed.
+
+ """
+
+ def__init__(
+ self,
+ port=21,
+ username="anonymous",
+ password="",
+ account="",
+ timeout=None,
+ progressbar=False,
+ chunk_size=1024,
+ ):
+ self.port=port
+ self.username=username
+ self.password=password
+ self.account=account
+ self.timeout=timeout
+ self.progressbar=progressbar
+ self.chunk_size=chunk_size
+ ifself.progressbarisTrueandtqdmisNone:
+ raiseValueError("Missing package 'tqdm' required for progress bars.")
+
+
+[docs]
+ def__call__(self,url,output_file,pooch,check_only=False):
+"""
+ Download the given URL over FTP to the given output file.
+
+ Parameters
+ ----------
+ url : str
+ The URL to the file you want to download.
+ output_file : str or file-like object
+ Path (and file name) to which the file will be downloaded.
+ pooch : :class:`~pooch.Pooch`
+ The instance of :class:`~pooch.Pooch` that is calling this method.
+ check_only : bool
+ If True, will only check if a file exists on the server and
+ **without downloading the file**. Will return ``True`` if the file
+ exists and ``False`` otherwise.
+
+ Returns
+ -------
+ availability : bool or None
+ If ``check_only==True``, returns a boolean indicating if the file
+ is available on the server. Otherwise, returns ``None``.
+
+ """
+ parsed_url=parse_url(url)
+ ftp=ftplib.FTP(timeout=self.timeout)
+ ftp.connect(host=parsed_url["netloc"],port=self.port)
+
+ ifcheck_only:
+ directory,file_name=os.path.split(parsed_url["path"])
+ try:
+ ftp.login(user=self.username,passwd=self.password,acct=self.account)
+ available=file_nameinftp.nlst(directory)
+ finally:
+ ftp.close()
+ returnavailable
+
+ ispath=nothasattr(output_file,"write")
+ ifispath:
+ # pylint: disable=consider-using-with
+ output_file=open(output_file,"w+b")
+ # pylint: enable=consider-using-with
+ try:
+ ftp.login(user=self.username,passwd=self.password,acct=self.account)
+ command=f"RETR {parsed_url['path']}"
+ ifself.progressbar:
+ # Make sure the file is set to binary mode, otherwise we can't
+ # get the file size. See: https://stackoverflow.com/a/22093848
+ ftp.voidcmd("TYPE I")
+ use_ascii=bool(sys.platform=="win32")
+ progress=tqdm(
+ total=int(ftp.size(parsed_url["path"])),
+ ncols=79,
+ ascii=use_ascii,
+ unit="B",
+ unit_scale=True,
+ leave=True,
+ )
+ withprogress:
+
+ defcallback(data):
+ "Update the progress bar and write to output"
+ progress.update(len(data))
+ output_file.write(data)
+
+ ftp.retrbinary(command,callback,blocksize=self.chunk_size)
+ else:
+ ftp.retrbinary(command,output_file.write,blocksize=self.chunk_size)
+ finally:
+ ftp.quit()
+ ifispath:
+ output_file.close()
+ returnNone
+
+
+
+
+
+[docs]
+classSFTPDownloader:# pylint: disable=too-few-public-methods
+"""
+ Download manager for fetching files over SFTP.
+
+ When called, downloads the given file URL into the specified local file.
+ Requires `paramiko <https://github.com/paramiko/paramiko>`__ to be
+ installed.
+
+ Use with :meth:`pooch.Pooch.fetch` or :func:`pooch.retrieve` to customize
+ the download of files (for example, to use authentication or print a
+ progress bar).
+
+ Parameters
+ ----------
+ port : int
+ Port used for the SFTP connection.
+ username : str
+ User name used to login to the server. Only needed if the server
+ requires authentication (i.e., no anonymous SFTP).
+ password : str
+ Password used to login to the server. Only needed if the server
+ requires authentication (i.e., no anonymous SFTP). Use the empty
+ string to indicate no password is required.
+ timeout : int
+ Timeout in seconds for sftp socket operations, use None to mean no
+ timeout.
+ progressbar : bool or an arbitrary progress bar object
+ If True, will print a progress bar of the download to standard
+ error (stderr). Requires `tqdm <https://github.com/tqdm/tqdm>`__ to
+ be installed.
+
+ """
+
+ def__init__(
+ self,
+ port=22,
+ username="anonymous",
+ password="",
+ account="",
+ timeout=None,
+ progressbar=False,
+ ):
+ self.port=port
+ self.username=username
+ self.password=password
+ self.account=account
+ self.timeout=timeout
+ self.progressbar=progressbar
+ # Collect errors and raise only once so that both missing packages are
+ # captured. Otherwise, the user is only warned of one of them at a
+ # time (and we can't test properly when they are both missing).
+ errors=[]
+ ifself.progressbarandtqdmisNone:
+ errors.append("Missing package 'tqdm' required for progress bars.")
+ ifparamikoisNone:
+ errors.append("Missing package 'paramiko' required for SFTP downloads.")
+ iferrors:
+ raiseValueError(" ".join(errors))
+
+
+[docs]
+ def__call__(self,url,output_file,pooch):
+"""
+ Download the given URL over SFTP to the given output file.
+
+ The output file must be given as a string (file name/path) and not an
+ open file object! Otherwise, paramiko cannot save to that file.
+
+ Parameters
+ ----------
+ url : str
+ The URL to the file you want to download.
+ output_file : str
+ Path (and file name) to which the file will be downloaded. **Cannot
+ be a file object**.
+ pooch : :class:`~pooch.Pooch`
+ The instance of :class:`~pooch.Pooch` that is calling this method.
+ """
+ parsed_url=parse_url(url)
+ connection=paramiko.Transport(sock=(parsed_url["netloc"],self.port))
+ sftp=None
+ try:
+ connection.connect(username=self.username,password=self.password)
+ sftp=paramiko.SFTPClient.from_transport(connection)
+ sftp.get_channel().settimeout=self.timeout
+ ifself.progressbar:
+ size=int(sftp.stat(parsed_url["path"]).st_size)
+ use_ascii=bool(sys.platform=="win32")
+ progress=tqdm(
+ total=size,
+ ncols=79,
+ ascii=use_ascii,
+ unit="B",
+ unit_scale=True,
+ leave=True,
+ )
+ ifself.progressbar:
+ withprogress:
+
+ defcallback(current,total):
+ "Update the progress bar and write to output"
+ progress.total=int(total)
+ progress.update(int(current-progress.n))
+
+ sftp.get(parsed_url["path"],output_file,callback=callback)
+ else:
+ sftp.get(parsed_url["path"],output_file)
+ finally:
+ connection.close()
+ ifsftpisnotNone:
+ sftp.close()
+
+
+
+
+
+[docs]
+classDOIDownloader:# pylint: disable=too-few-public-methods
+"""
+ Download manager for fetching files from Digital Object Identifiers (DOIs).
+
+ Open-access data repositories often issue Digital Object Identifiers (DOIs)
+ for data which provide a stable link and citation point. The trick is
+ finding out the download URL for a file given the DOI.
+
+ When called, this downloader uses the repository's public API to find out
+ the download URL from the DOI and file name. It then uses
+ :class:`pooch.HTTPDownloader` to download the URL into the specified local
+ file. Allowing "URL"s to be specified with the DOI instead of the actual
+ HTTP download link. Uses the :mod:`requests` library to manage downloads
+ and interact with the APIs.
+
+ The **format of the "URL"** is: ``doi:{DOI}/{file name}``.
+
+ Notice that there are no ``//`` like in HTTP/FTP and you must specify a
+ file name after the DOI (separated by a ``/``).
+
+ Use with :meth:`pooch.Pooch.fetch` or :func:`pooch.retrieve` to be able to
+ download files given the DOI instead of an HTTP link.
+
+ Supported repositories:
+
+ * `figshare <https://www.figshare.com>`__
+ * `Zenodo <https://www.zenodo.org>`__
+ * `Dataverse <https://dataverse.org/>`__ instances
+
+ .. attention::
+
+ DOIs from other repositories **will not work** since we need to access
+ their particular APIs to find the download links. We welcome
+ suggestions and contributions adding new repositories.
+
+ Parameters
+ ----------
+ progressbar : bool or an arbitrary progress bar object
+ If True, will print a progress bar of the download to standard error
+ (stderr). Requires `tqdm <https://github.com/tqdm/tqdm>`__ to be
+ installed. Alternatively, an arbitrary progress bar object can be
+ passed. See :ref:`custom-progressbar` for details.
+ chunk_size : int
+ Files are streamed *chunk_size* bytes at a time instead of loading
+ everything into memory at one. Usually doesn't need to be changed.
+ **kwargs
+ All keyword arguments given when creating an instance of this class
+ will be passed to :func:`requests.get`.
+
+ Examples
+ --------
+
+ Download one of the data files from the figshare archive of Pooch test
+ data:
+
+ >>> import os
+ >>> downloader = DOIDownloader()
+ >>> url = "doi:10.6084/m9.figshare.14763051.v1/tiny-data.txt"
+ >>> # Not using with Pooch.fetch so no need to pass an instance of Pooch
+ >>> downloader(url=url, output_file="tiny-data.txt", pooch=None)
+ >>> os.path.exists("tiny-data.txt")
+ True
+ >>> with open("tiny-data.txt") as f:
+ ... print(f.read().strip())
+ # A tiny data file for test purposes only
+ 1 2 3 4 5 6
+ >>> os.remove("tiny-data.txt")
+
+ Same thing but for our Zenodo archive:
+
+ >>> url = "doi:10.5281/zenodo.4924875/tiny-data.txt"
+ >>> downloader(url=url, output_file="tiny-data.txt", pooch=None)
+ >>> os.path.exists("tiny-data.txt")
+ True
+ >>> with open("tiny-data.txt") as f:
+ ... print(f.read().strip())
+ # A tiny data file for test purposes only
+ 1 2 3 4 5 6
+ >>> os.remove("tiny-data.txt")
+
+ """
+
+ def__init__(self,progressbar=False,chunk_size=1024,**kwargs):
+ self.kwargs=kwargs
+ self.progressbar=progressbar
+ self.chunk_size=chunk_size
+
+
+[docs]
+ def__call__(self,url,output_file,pooch):
+"""
+ Download the given DOI URL over HTTP to the given output file.
+
+ Uses the repository's API to determine the actual HTTP download URL
+ from the given DOI.
+
+ Uses :func:`requests.get`.
+
+ Parameters
+ ----------
+ url : str
+ The URL to the file you want to download.
+ output_file : str or file-like object
+ Path (and file name) to which the file will be downloaded.
+ pooch : :class:`~pooch.Pooch`
+ The instance of :class:`~pooch.Pooch` that is calling this method.
+
+ """
+
+ parsed_url=parse_url(url)
+ data_repository=doi_to_repository(parsed_url["netloc"])
+
+ # Resolve the URL
+ file_name=parsed_url["path"]
+ # remove the leading slash in the path
+ iffile_name[0]=="/":
+ file_name=file_name[1:]
+ download_url=data_repository.download_url(file_name)
+
+ # Instantiate the downloader object
+ downloader=HTTPDownloader(
+ progressbar=self.progressbar,chunk_size=self.chunk_size,**self.kwargs
+ )
+ downloader(download_url,output_file,pooch)
+
+
+
+
+defdoi_to_url(doi):
+"""
+ Follow a DOI link to resolve the URL of the archive.
+
+ Parameters
+ ----------
+ doi : str
+ The DOI of the archive.
+
+ Returns
+ -------
+ url : str
+ The URL of the archive in the data repository.
+
+ """
+ # Lazy import requests to speed up import time
+ importrequests# pylint: disable=C0415
+
+ # Use doi.org to resolve the DOI to the repository website.
+ response=requests.get(f"https://doi.org/{doi}",timeout=5)
+ url=response.url
+ if400<=response.status_code<600:
+ raiseValueError(
+ f"Archive with doi:{doi} not found (see {url}). Is the DOI correct?"
+ )
+ returnurl
+
+
+defdoi_to_repository(doi):
+"""
+ Instantiate a data repository instance from a given DOI.
+
+ This function implements the chain of responsibility dispatch
+ to the correct data repository class.
+
+ Parameters
+ ----------
+ doi : str
+ The DOI of the archive.
+
+ Returns
+ -------
+ data_repository : DataRepository
+ The data repository object
+ """
+
+ # This should go away in a separate issue: DOI handling should
+ # not rely on the (non-)existence of trailing slashes. The issue
+ # is documented in https://github.com/fatiando/pooch/issues/324
+ ifdoi[-1]=="/":
+ doi=doi[:-1]
+
+ repositories=[
+ FigshareRepository,
+ ZenodoRepository,
+ DataverseRepository,
+ ]
+
+ # Extract the DOI and the repository information
+ archive_url=doi_to_url(doi)
+
+ # Try the converters one by one until one of them returned a URL
+ data_repository=None
+ forrepoinrepositories:
+ ifdata_repositoryisNone:
+ data_repository=repo.initialize(
+ archive_url=archive_url,
+ doi=doi,
+ )
+
+ ifdata_repositoryisNone:
+ repository=parse_url(archive_url)["netloc"]
+ raiseValueError(
+ f"Invalid data repository '{repository}'. "
+ "To request or contribute support for this repository, "
+ "please open an issue at https://github.com/fatiando/pooch/issues"
+ )
+
+ returndata_repository
+
+
+classDataRepository:# pylint: disable=too-few-public-methods, missing-class-docstring
+ @classmethod
+ definitialize(cls,doi,archive_url):# pylint: disable=unused-argument
+"""
+ Initialize the data repository if the given URL points to a
+ corresponding repository.
+
+ Initializes a data repository object. This is done as part of
+ a chain of responsibility. If the class cannot handle the given
+ repository URL, it returns `None`. Otherwise a `DataRepository`
+ instance is returned.
+
+ Parameters
+ ----------
+ doi : str
+ The DOI that identifies the repository
+ archive_url : str
+ The resolved URL for the DOI
+ """
+
+ returnNone# pragma: no cover
+
+ defdownload_url(self,file_name):
+"""
+ Use the repository API to get the download URL for a file given
+ the archive URL.
+
+ Parameters
+ ----------
+ file_name : str
+ The name of the file in the archive that will be downloaded.
+
+ Returns
+ -------
+ download_url : str
+ The HTTP URL that can be used to download the file.
+ """
+
+ raiseNotImplementedError# pragma: no cover
+
+ defpopulate_registry(self,pooch):
+"""
+ Populate the registry using the data repository's API
+
+ Parameters
+ ----------
+ pooch : Pooch
+ The pooch instance that the registry will be added to.
+ """
+
+ raiseNotImplementedError# pragma: no cover
+
+
+classZenodoRepository(DataRepository):# pylint: disable=missing-class-docstring
+ base_api_url="https://zenodo.org/api/records"
+
+ def__init__(self,doi,archive_url):
+ self.archive_url=archive_url
+ self.doi=doi
+ self._api_response=None
+ self._api_version=None
+
+ @classmethod
+ definitialize(cls,doi,archive_url):
+"""
+ Initialize the data repository if the given URL points to a
+ corresponding repository.
+
+ Initializes a data repository object. This is done as part of
+ a chain of responsibility. If the class cannot handle the given
+ repository URL, it returns `None`. Otherwise a `DataRepository`
+ instance is returned.
+
+ Parameters
+ ----------
+ doi : str
+ The DOI that identifies the repository
+ archive_url : str
+ The resolved URL for the DOI
+ """
+
+ # Check whether this is a Zenodo URL
+ parsed_archive_url=parse_url(archive_url)
+ ifparsed_archive_url["netloc"]!="zenodo.org":
+ returnNone
+
+ returncls(doi,archive_url)
+
+ @property
+ defapi_response(self):
+"""Cached API response from Zenodo"""
+ ifself._api_responseisNone:
+ # Lazy import requests to speed up import time
+ importrequests# pylint: disable=C0415
+
+ article_id=self.archive_url.split("/")[-1]
+ self._api_response=requests.get(
+ f"{self.base_api_url}/{article_id}",
+ timeout=5,
+ ).json()
+
+ returnself._api_response
+
+ @property
+ defapi_version(self):
+"""
+ Version of the Zenodo API we are interacting with
+
+ The versions can either be :
+
+ - ``"legacy"``: corresponds to the Zenodo API that was supported until
+ 2023-10-12 (before the migration to InvenioRDM).
+ - ``"new"``: corresponds to the new API that went online on 2023-10-13
+ after the migration to InvenioRDM.
+
+ The ``"new"`` API breaks backward compatibility with the ``"legacy"``
+ one and could probably be replaced by an updated version that restores
+ the behaviour of the ``"legacy"`` one.
+
+ Returns
+ -------
+ str
+ """
+ ifself._api_versionisNone:
+ ifall("key"infileforfileinself.api_response["files"]):
+ self._api_version="legacy"
+ elifall("filename"infileforfileinself.api_response["files"]):
+ self._api_version="new"
+ else:
+ raiseValueError(
+ "Couldn't determine the version of the Zenodo API for "
+ f"{self.archive_url} (doi:{self.doi})."
+ )
+ returnself._api_version
+
+ defdownload_url(self,file_name):
+"""
+ Use the repository API to get the download URL for a file given
+ the archive URL.
+
+ Parameters
+ ----------
+ file_name : str
+ The name of the file in the archive that will be downloaded.
+
+ Returns
+ -------
+ download_url : str
+ The HTTP URL that can be used to download the file.
+
+ Notes
+ -----
+ After Zenodo migrated to InvenioRDM on Oct 2023, their API changed. The
+ link to the desired files that appears in the API response leads to 404
+ errors (by 2023-10-17). The files are available in the following url:
+ ``https://zenodo.org/records/{article_id}/files/{file_name}?download=1``.
+
+ This method supports both the legacy and the new API.
+ """
+ # Create list of files in the repository
+ ifself.api_version=="legacy":
+ files={item["key"]:itemforiteminself.api_response["files"]}
+ else:
+ files=[item["filename"]foriteminself.api_response["files"]]
+ # Check if file exists in the repository
+ iffile_namenotinfiles:
+ raiseValueError(
+ f"File '{file_name}' not found in data archive "
+ f"{self.archive_url} (doi:{self.doi})."
+ )
+ # Build download url
+ ifself.api_version=="legacy":
+ download_url=files[file_name]["links"]["self"]
+ else:
+ article_id=self.api_response["id"]
+ download_url=(
+ f"https://zenodo.org/records/{article_id}/files/{file_name}?download=1"
+ )
+ returndownload_url
+
+ defpopulate_registry(self,pooch):
+"""
+ Populate the registry using the data repository's API
+
+ Parameters
+ ----------
+ pooch : Pooch
+ The pooch instance that the registry will be added to.
+
+ Notes
+ -----
+ After Zenodo migrated to InvenioRDM on Oct 2023, their API changed. The
+ checksums for each file listed in the API reference is now an md5 sum.
+
+ This method supports both the legacy and the new API.
+ """
+ forfiledatainself.api_response["files"]:
+ checksum=filedata["checksum"]
+ ifself.api_version=="legacy":
+ key="key"
+ else:
+ key="filename"
+ checksum=f"md5:{checksum}"
+ pooch.registry[filedata[key]]=checksum
+
+
+classFigshareRepository(DataRepository):# pylint: disable=missing-class-docstring
+ def__init__(self,doi,archive_url):
+ self.archive_url=archive_url
+ self.doi=doi
+ self._api_response=None
+
+ @classmethod
+ definitialize(cls,doi,archive_url):
+"""
+ Initialize the data repository if the given URL points to a
+ corresponding repository.
+
+ Initializes a data repository object. This is done as part of
+ a chain of responsibility. If the class cannot handle the given
+ repository URL, it returns `None`. Otherwise a `DataRepository`
+ instance is returned.
+
+ Parameters
+ ----------
+ doi : str
+ The DOI that identifies the repository
+ archive_url : str
+ The resolved URL for the DOI
+ """
+
+ # Check whether this is a Figshare URL
+ parsed_archive_url=parse_url(archive_url)
+ ifparsed_archive_url["netloc"]!="figshare.com":
+ returnNone
+
+ returncls(doi,archive_url)
+
+ def_parse_version_from_doi(self):
+"""
+ Parse version from the doi
+
+ Return None if version is not available in the doi.
+ """
+ # Get suffix of the doi
+ _,suffix=self.doi.split("/")
+ # Split the suffix by dots and keep the last part
+ last_part=suffix.split(".")[-1]
+ # Parse the version from the last part
+ iflast_part[0]!="v":
+ returnNone
+ version=int(last_part[1:])
+ returnversion
+
+ @property
+ defapi_response(self):
+"""Cached API response from Figshare"""
+ ifself._api_responseisNone:
+ # Lazy import requests to speed up import time
+ importrequests# pylint: disable=C0415
+
+ # Use the figshare API to find the article ID from the DOI
+ article=requests.get(
+ f"https://api.figshare.com/v2/articles?doi={self.doi}",
+ timeout=5,
+ ).json()[0]
+ article_id=article["id"]
+ # Parse desired version from the doi
+ version=self._parse_version_from_doi()
+ # With the ID and version, we can get a list of files and their
+ # download links
+ ifversionisNone:
+ # Figshare returns the latest version available when no version
+ # is specified through the DOI.
+ warnings.warn(
+ f"The Figshare DOI '{self.doi}' doesn't specify which version of "
+ "the repository should be used. "
+ "Figshare will point to the latest version available.",
+ UserWarning,
+ )
+ # Define API url using only the article id
+ # (figshare will resolve the latest version)
+ api_url=f"https://api.figshare.com/v2/articles/{article_id}"
+ else:
+ # Define API url using article id and the desired version
+ # Get list of files using article id and the version
+ api_url=(
+ "https://api.figshare.com/v2/articles/"
+ f"{article_id}/versions/{version}"
+ )
+ # Make the request and return the files in the figshare repository
+ response=requests.get(api_url,timeout=5)
+ response.raise_for_status()
+ self._api_response=response.json()["files"]
+
+ returnself._api_response
+
+ defdownload_url(self,file_name):
+"""
+ Use the repository API to get the download URL for a file given
+ the archive URL.
+
+ Parameters
+ ----------
+ file_name : str
+ The name of the file in the archive that will be downloaded.
+
+ Returns
+ -------
+ download_url : str
+ The HTTP URL that can be used to download the file.
+ """
+ files={item["name"]:itemforiteminself.api_response}
+ iffile_namenotinfiles:
+ raiseValueError(
+ f"File '{file_name}' not found in data archive {self.archive_url} (doi:{self.doi})."
+ )
+ download_url=files[file_name]["download_url"]
+ returndownload_url
+
+ defpopulate_registry(self,pooch):
+"""
+ Populate the registry using the data repository's API
+
+ Parameters
+ ----------
+ pooch : Pooch
+ The pooch instance that the registry will be added to.
+ """
+
+ forfiledatainself.api_response:
+ pooch.registry[filedata["name"]]=f"md5:{filedata['computed_md5']}"
+
+
+classDataverseRepository(DataRepository):# pylint: disable=missing-class-docstring
+ def__init__(self,doi,archive_url):
+ self.archive_url=archive_url
+ self.doi=doi
+ self._api_response=None
+
+ @classmethod
+ definitialize(cls,doi,archive_url):
+"""
+ Initialize the data repository if the given URL points to a
+ corresponding repository.
+
+ Initializes a data repository object. This is done as part of
+ a chain of responsibility. If the class cannot handle the given
+ repository URL, it returns `None`. Otherwise a `DataRepository`
+ instance is returned.
+
+ Parameters
+ ----------
+ doi : str
+ The DOI that identifies the repository
+ archive_url : str
+ The resolved URL for the DOI
+ """
+ # Access the DOI as if this was a DataVerse instance
+ response=cls._get_api_response(doi,archive_url)
+
+ # If we failed, this is probably not a DataVerse instance
+ if400<=response.status_code<600:
+ returnNone
+
+ # Initialize the repository and overwrite the api response
+ repository=cls(doi,archive_url)
+ repository.api_response=response
+ returnrepository
+
+ @classmethod
+ def_get_api_response(cls,doi,archive_url):
+"""
+ Perform the actual API request
+
+ This has been separated into a separate ``classmethod``, as it can be
+ used prior and after the initialization.
+ """
+ # Lazy import requests to speed up import time
+ importrequests# pylint: disable=C0415
+
+ parsed=parse_url(archive_url)
+ response=requests.get(
+ f"{parsed['protocol']}://{parsed['netloc']}/api/datasets/"
+ f":persistentId?persistentId=doi:{doi}",
+ timeout=5,
+ )
+ returnresponse
+
+ @property
+ defapi_response(self):
+"""Cached API response from a DataVerse instance"""
+
+ ifself._api_responseisNone:
+ self._api_response=self._get_api_response(
+ self.doi,self.archive_url
+ )# pragma: no cover
+
+ returnself._api_response
+
+ @api_response.setter
+ defapi_response(self,response):
+"""Update the cached API response"""
+
+ self._api_response=response
+
+ defdownload_url(self,file_name):
+"""
+ Use the repository API to get the download URL for a file given
+ the archive URL.
+
+ Parameters
+ ----------
+ file_name : str
+ The name of the file in the archive that will be downloaded.
+
+ Returns
+ -------
+ download_url : str
+ The HTTP URL that can be used to download the file.
+ """
+ parsed=parse_url(self.archive_url)
+ response=self.api_response.json()
+ files={
+ file["dataFile"]["filename"]:file["dataFile"]
+ forfileinresponse["data"]["latestVersion"]["files"]
+ }
+ iffile_namenotinfiles:
+ raiseValueError(
+ f"File '{file_name}' not found in data archive "
+ f"{self.archive_url} (doi:{self.doi})."
+ )
+ # Generate download_url using the file id
+ download_url=(
+ f"{parsed['protocol']}://{parsed['netloc']}/api/access/datafile/"
+ f"{files[file_name]['id']}"
+ )
+ returndownload_url
+
+ defpopulate_registry(self,pooch):
+"""
+ Populate the registry using the data repository's API
+
+ Parameters
+ ----------
+ pooch : Pooch
+ The pooch instance that the registry will be added to.
+ """
+
+ forfiledatainself.api_response.json()["data"]["latestVersion"]["files"]:
+ pooch.registry[filedata["dataFile"]["filename"]]=(
+ f"md5:{filedata['dataFile']['md5']}"
+ )
+
+# Copyright (c) 2018 The Pooch Developers.
+# Distributed under the terms of the BSD 3-Clause License.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# This code is part of the Fatiando a Terra project (https://www.fatiando.org)
+#
+"""
+Calculating and checking file hashes.
+"""
+importhashlib
+importfunctools
+frompathlibimportPath
+
+# From the docs: https://docs.python.org/3/library/hashlib.html#hashlib.new
+# The named constructors are much faster than new() and should be
+# preferred.
+# Need to fallback on new() for some algorithms.
+ALGORITHMS_AVAILABLE={
+ alg:getattr(hashlib,alg,functools.partial(hashlib.new,alg))
+ foralginhashlib.algorithms_available
+}
+
+try:
+ importxxhash
+
+ # xxhash doesn't have a list of available algorithms yet.
+ # https://github.com/ifduyue/python-xxhash/issues/48
+ ALGORITHMS_AVAILABLE.update(
+ {
+ alg:getattr(xxhash,alg,None)
+ foralgin["xxh128","xxh64","xxh32","xxh3_128","xxh3_64"]
+ }
+ )
+ # The xxh3 algorithms are only available for version>=2.0. Set to None and
+ # remove to ensure backwards compatibility.
+ ALGORITHMS_AVAILABLE={
+ alg:funcforalg,funcinALGORITHMS_AVAILABLE.items()iffuncisnotNone
+ }
+exceptImportError:
+ pass
+
+
+
+[docs]
+deffile_hash(fname,alg="sha256"):
+"""
+ Calculate the hash of a given file.
+
+ Useful for checking if a file has changed or been corrupted.
+
+ Parameters
+ ----------
+ fname : str
+ The name of the file.
+ alg : str
+ The type of the hashing algorithm
+
+ Returns
+ -------
+ hash : str
+ The hash of the file.
+
+ Examples
+ --------
+
+ >>> fname = "test-file-for-hash.txt"
+ >>> with open(fname, "w") as f:
+ ... __ = f.write("content of the file")
+ >>> print(file_hash(fname))
+ 0fc74468e6a9a829f103d069aeb2bb4f8646bad58bf146bb0e3379b759ec4a00
+ >>> import os
+ >>> os.remove(fname)
+
+ """
+ ifalgnotinALGORITHMS_AVAILABLE:
+ raiseValueError(
+ f"Algorithm '{alg}' not available to the pooch library. "
+ "Only the following algorithms are available "
+ f"{list(ALGORITHMS_AVAILABLE.keys())}."
+ )
+ # Calculate the hash in chunks to avoid overloading the memory
+ chunksize=65536
+ hasher=ALGORITHMS_AVAILABLE[alg]()
+ withopen(fname,"rb")asfin:
+ buff=fin.read(chunksize)
+ whilebuff:
+ hasher.update(buff)
+ buff=fin.read(chunksize)
+ returnhasher.hexdigest()
+
+
+
+defhash_algorithm(hash_string):
+"""
+ Parse the name of the hash method from the hash string.
+
+ The hash string should have the following form ``algorithm:hash``, where
+ algorithm can be the name of any algorithm known to :mod:`hashlib`.
+
+ If the algorithm is omitted or the hash string is None, will default to
+ ``"sha256"``.
+
+ Parameters
+ ----------
+ hash_string : str
+ The hash string with optional algorithm prepended.
+
+ Returns
+ -------
+ hash_algorithm : str
+ The name of the algorithm.
+
+ Examples
+ --------
+
+ >>> print(hash_algorithm("qouuwhwd2j192y1lb1iwgowdj2898wd2d9"))
+ sha256
+ >>> print(hash_algorithm("md5:qouuwhwd2j192y1lb1iwgowdj2898wd2d9"))
+ md5
+ >>> print(hash_algorithm("sha256:qouuwhwd2j192y1lb1iwgowdj2898wd2d9"))
+ sha256
+ >>> print(hash_algorithm("SHA256:qouuwhwd2j192y1lb1iwgowdj2898wd2d9"))
+ sha256
+ >>> print(hash_algorithm("xxh3_64:qouuwhwd2j192y1lb1iwgowdj2898wd2d9"))
+ xxh3_64
+ >>> print(hash_algorithm(None))
+ sha256
+
+ """
+ default="sha256"
+ ifhash_stringisNone:
+ algorithm=default
+ elif":"notinhash_string:
+ algorithm=default
+ else:
+ algorithm=hash_string.split(":")[0]
+ returnalgorithm.lower()
+
+
+defhash_matches(fname,known_hash,strict=False,source=None):
+"""
+ Check if the hash of a file matches a known hash.
+
+ If the *known_hash* is None, will always return True.
+
+ Coverts hashes to lowercase before comparison to avoid system specific
+ mismatches between hashes in the registry and computed hashes.
+
+ Parameters
+ ----------
+ fname : str or PathLike
+ The path to the file.
+ known_hash : str
+ The known hash. Optionally, prepend ``alg:`` to the hash to specify the
+ hashing algorithm. Default is SHA256.
+ strict : bool
+ If True, will raise a :class:`ValueError` if the hash does not match
+ informing the user that the file may be corrupted.
+ source : str
+ The source of the downloaded file (name or URL, for example). Will be
+ used in the error message if *strict* is True. Has no other use other
+ than reporting to the user where the file came from in case of hash
+ mismatch. If None, will default to *fname*.
+
+ Returns
+ -------
+ is_same : bool
+ True if the hash matches, False otherwise.
+
+ """
+ ifknown_hashisNone:
+ returnTrue
+ algorithm=hash_algorithm(known_hash)
+ new_hash=file_hash(fname,alg=algorithm)
+ matches=new_hash.lower()==known_hash.split(":")[-1].lower()
+ ifstrictandnotmatches:
+ ifsourceisNone:
+ source=str(fname)
+ raiseValueError(
+ f"{algorithm.upper()} hash of downloaded file ({source}) does not match"
+ f" the known hash: expected {known_hash} but got {new_hash}. Deleted"
+ " download for safety. The downloaded file may have been corrupted or"
+ " the known hash may be outdated."
+ )
+ returnmatches
+
+
+
+[docs]
+defmake_registry(directory,output,recursive=True):
+"""
+ Make a registry of files and hashes for the given directory.
+
+ This is helpful if you have many files in your test dataset as it keeps you
+ from needing to manually update the registry.
+
+ Parameters
+ ----------
+ directory : str
+ Directory of the test data to put in the registry. All file names in
+ the registry will be relative to this directory.
+ output : str
+ Name of the output registry file.
+ recursive : bool
+ If True, will recursively look for files in subdirectories of
+ *directory*.
+
+ """
+ directory=Path(directory)
+ ifrecursive:
+ pattern="**/*"
+ else:
+ pattern="*"
+
+ files=sorted(
+ str(path.relative_to(directory))
+ forpathindirectory.glob(pattern)
+ ifpath.is_file()
+ )
+
+ hashes=[file_hash(str(directory/fname))forfnameinfiles]
+
+ withopen(output,"w",encoding="utf-8")asoutfile:
+ forfname,fhashinzip(files,hashes):
+ # Only use Unix separators for the registry so that we don't go
+ # insane dealing with file paths.
+ outfile.write("{}{}\n".format(fname.replace("\\","/"),fhash))
+# Copyright (c) 2018 The Pooch Developers.
+# Distributed under the terms of the BSD 3-Clause License.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# This code is part of the Fatiando a Terra project (https://www.fatiando.org)
+#
+# pylint: disable=line-too-long
+"""
+Post-processing hooks
+"""
+importabc
+importos
+importbz2
+importgzip
+importlzma
+importshutil
+fromzipfileimportZipFile
+fromtarfileimportTarFile
+
+from.utilsimportget_logger
+
+
+classExtractorProcessor(abc.ABC):# pylint: disable=too-few-public-methods
+"""
+ Abstract base class for extractions from compressed archives.
+
+ Subclasses can be used with :meth:`pooch.Pooch.fetch` and
+ :func:`pooch.retrieve` to unzip a downloaded data file into a folder in the
+ local data store. :meth:`~pooch.Pooch.fetch` will return a list with the
+ names of the extracted files instead of the archive.
+
+ Parameters
+ ----------
+ members : list or None
+ If None, will unpack all files in the archive. Otherwise, *members*
+ must be a list of file names to unpack from the archive. Only these
+ files will be unpacked.
+ extract_dir : str or None
+ If None, files will be unpacked to the default location (a folder in
+ the same location as the downloaded zip file, with a suffix added).
+ Otherwise, files will be unpacked to ``extract_dir``, which is
+ interpreted as a *relative path* (relative to the cache location
+ provided by :func:`pooch.retrieve` or :meth:`pooch.Pooch.fetch`).
+
+ """
+
+ def__init__(self,members=None,extract_dir=None):
+ self.members=members
+ self.extract_dir=extract_dir
+
+ @property
+ @abc.abstractmethod
+ defsuffix(self):
+"""
+ String appended to unpacked archive folder name.
+ Only used if extract_dir is None.
+ MUST BE IMPLEMENTED BY CHILD CLASSES.
+ """
+
+ @abc.abstractmethod
+ def_all_members(self,fname):
+"""
+ Return all the members in the archive.
+ MUST BE IMPLEMENTED BY CHILD CLASSES.
+ """
+
+ @abc.abstractmethod
+ def_extract_file(self,fname,extract_dir):
+"""
+ This method receives an argument for the archive to extract and the
+ destination path.
+ MUST BE IMPLEMENTED BY CHILD CLASSES.
+ """
+
+ def__call__(self,fname,action,pooch):
+"""
+ Extract all files from the given archive.
+
+ Parameters
+ ----------
+ fname : str
+ Full path of the zipped file in local storage.
+ action : str
+ Indicates what action was taken by :meth:`pooch.Pooch.fetch` or
+ :func:`pooch.retrieve`:
+
+ * ``"download"``: File didn't exist locally and was downloaded
+ * ``"update"``: Local file was outdated and was re-download
+ * ``"fetch"``: File exists and is updated so it wasn't downloaded
+
+ pooch : :class:`pooch.Pooch`
+ The instance of :class:`pooch.Pooch` that is calling this.
+
+ Returns
+ -------
+ fnames : list of str
+ A list of the full path to all files in the extracted archive.
+
+ """
+ ifself.extract_dirisNone:
+ self.extract_dir=fname+self.suffix
+ else:
+ archive_dir=fname.rsplit(os.path.sep,maxsplit=1)[0]
+ self.extract_dir=os.path.join(archive_dir,self.extract_dir)
+ # Get a list of everyone who is supposed to be in the unpacked folder
+ # so we can check if they are all there or if we need to extract new
+ # files.
+ ifself.membersisNoneornotself.members:
+ members=self._all_members(fname)
+ else:
+ members=self.members
+ if(
+ (actionin("update","download"))
+ or(notos.path.exists(self.extract_dir))
+ ornotall(
+ os.path.exists(os.path.join(self.extract_dir,m))forminmembers
+ )
+ ):
+ # Make sure that the folder with the extracted files exists
+ os.makedirs(self.extract_dir,exist_ok=True)
+ self._extract_file(fname,self.extract_dir)
+
+ # Get a list of all file names (including subdirectories) in our folder
+ # of unzipped files, filtered by the given members list
+ fnames=[]
+ forpath,_,filesinos.walk(self.extract_dir):
+ forfilenameinfiles:
+ relpath=os.path.normpath(
+ os.path.join(os.path.relpath(path,self.extract_dir),filename)
+ )
+ ifself.membersisNoneorany(
+ relpath.startswith(os.path.normpath(m))forminself.members
+ ):
+ fnames.append(os.path.join(path,filename))
+
+ returnfnames
+
+
+
+[docs]
+classUnzip(ExtractorProcessor):# pylint: disable=too-few-public-methods
+"""
+ Processor that unpacks a zip archive and returns a list of all files.
+
+ Use with :meth:`pooch.Pooch.fetch` or :func:`pooch.retrieve` to unzip a
+ downloaded data file into a folder in the local data store. The
+ method/function will return a list with the names of the unzipped files
+ instead of the zip archive.
+
+ The output folder is ``{fname}.unzip``.
+
+ Parameters
+ ----------
+ members : list or None
+ If None, will unpack all files in the zip archive. Otherwise, *members*
+ must be a list of file names to unpack from the archive. Only these
+ files will be unpacked.
+ extract_dir : str or None
+ If None, files will be unpacked to the default location (a folder in
+ the same location as the downloaded zip file, with the suffix
+ ``.unzip`` added). Otherwise, files will be unpacked to
+ ``extract_dir``, which is interpreted as a *relative path* (relative to
+ the cache location provided by :func:`pooch.retrieve` or
+ :meth:`pooch.Pooch.fetch`).
+
+ """
+
+ @property
+ defsuffix(self):
+"""
+ String appended to unpacked archive folder name.
+ Only used if extract_dir is None.
+ """
+ return".unzip"
+
+ def_all_members(self,fname):
+"""Return all members from a given archive."""
+ withZipFile(fname,"r")aszip_file:
+ returnzip_file.namelist()
+
+ def_extract_file(self,fname,extract_dir):
+"""
+ This method receives an argument for the archive to extract and the
+ destination path.
+ """
+ withZipFile(fname,"r")aszip_file:
+ ifself.membersisNone:
+ get_logger().info(
+ "Unzipping contents of '%s' to '%s'",fname,extract_dir
+ )
+ # Unpack all files from the archive into our new folder
+ zip_file.extractall(path=extract_dir)
+ else:
+ formemberinself.members:
+ get_logger().info(
+ "Extracting '%s' from '%s' to '%s'",member,fname,extract_dir
+ )
+ # If the member is a dir, we need to get the names of the
+ # elements it contains for extraction (ZipFile does not
+ # support dirs on .extract). If it's not a dir, this will
+ # only include the member itself.
+ # Based on:
+ # https://stackoverflow.com/questions/8008829/extract-only-a-single-directory-from-tar
+ subdir_members=[
+ name
+ fornameinzip_file.namelist()
+ ifos.path.normpath(name).startswith(os.path.normpath(member))
+ ]
+ # Extract the data file from within the archive
+ zip_file.extractall(members=subdir_members,path=extract_dir)
+
+
+
+
+[docs]
+classUntar(ExtractorProcessor):# pylint: disable=too-few-public-methods
+"""
+ Processor that unpacks a tar archive and returns a list of all files.
+
+ Use with :meth:`pooch.Pooch.fetch` or :func:`pooch.retrieve` to untar a
+ downloaded data file into a folder in the local data store. The
+ method/function will return a list with the names of the extracted files
+ instead of the archive.
+
+ The output folder is ``{fname}.untar``.
+
+
+ Parameters
+ ----------
+ members : list or None
+ If None, will unpack all files in the archive. Otherwise, *members*
+ must be a list of file names to unpack from the archive. Only these
+ files will be unpacked.
+ extract_dir : str or None
+ If None, files will be unpacked to the default location (a folder in
+ the same location as the downloaded tar file, with the suffix
+ ``.untar`` added). Otherwise, files will be unpacked to
+ ``extract_dir``, which is interpreted as a *relative path* (relative to
+ the cache location provided by :func:`pooch.retrieve` or
+ :meth:`pooch.Pooch.fetch`).
+ """
+
+ @property
+ defsuffix(self):
+"""
+ String appended to unpacked archive folder name.
+ Only used if extract_dir is None.
+ """
+ return".untar"
+
+ def_all_members(self,fname):
+"""Return all members from a given archive."""
+ withTarFile.open(fname,"r")astar_file:
+ return[info.nameforinfointar_file.getmembers()]
+
+ def_extract_file(self,fname,extract_dir):
+"""
+ This method receives an argument for the archive to extract and the
+ destination path.
+ """
+ withTarFile.open(fname,"r")astar_file:
+ ifself.membersisNone:
+ get_logger().info(
+ "Untarring contents of '%s' to '%s'",fname,extract_dir
+ )
+ # Unpack all files from the archive into our new folder
+ tar_file.extractall(path=extract_dir)
+ else:
+ formemberinself.members:
+ get_logger().info(
+ "Extracting '%s' from '%s' to '%s'",member,fname,extract_dir
+ )
+ # If the member is a dir, we need to get the names of the
+ # elements it contains for extraction (TarFile does not
+ # support dirs on .extract). If it's not a dir, this will
+ # only include the member itself.
+ # Based on:
+ # https://stackoverflow.com/questions/8008829/extract-only-a-single-directory-from-tar
+ # Can't use .getnames because extractall expects TarInfo
+ # objects.
+ subdir_members=[
+ info
+ forinfointar_file.getmembers()
+ ifos.path.normpath(info.name).startswith(
+ os.path.normpath(member)
+ )
+ ]
+ # Extract the data file from within the archive
+ tar_file.extractall(members=subdir_members,path=extract_dir)
+
+
+
+
+[docs]
+classDecompress:# pylint: disable=too-few-public-methods
+"""
+ Processor that decompress a file and returns the decompressed version.
+
+ Use with :meth:`pooch.Pooch.fetch` or :func:`pooch.retrieve` to decompress
+ a downloaded data file so that it can be easily opened. Useful for data
+ files that take a long time to decompress (exchanging disk space for
+ speed).
+
+ Supported decompression methods are LZMA (``.xz``), bzip2 (``.bz2``), and
+ gzip (``.gz``).
+
+ File names with the standard extensions (see above) can use
+ ``method="auto"`` to automatically determine the compression method. This
+ can be overwritten by setting the *method* argument.
+
+ .. note::
+
+ To unpack zip and tar archives with one or more files, use
+ :class:`pooch.Unzip` and :class:`pooch.Untar` instead.
+
+ The output file is ``{fname}.decomp`` by default but it can be changed by
+ setting the ``name`` parameter.
+
+ .. warning::
+
+ Passing in ``name`` can cause existing data to be lost! For example, if
+ a file already exists with the specified name it will be overwritten
+ with the new decompressed file content. **Use this option with
+ caution.**
+
+ Parameters
+ ----------
+ method : str
+ Name of the compression method. Can be "auto", "lzma", "xz", "bzip2",
+ or "gzip".
+ name : None or str
+ Defines the decompressed file name. The file name will be
+ ``{fname}.decomp`` if ``None`` (default) or the given name otherwise.
+ Note that the name should **not** include the full (or relative) path,
+ it should be just the file name itself.
+
+ """
+
+ modules={"auto":None,"lzma":lzma,"xz":lzma,"gzip":gzip,"bzip2":bz2}
+ extensions={".xz":"lzma",".gz":"gzip",".bz2":"bzip2"}
+
+ def__init__(self,method="auto",name=None):
+ self.method=method
+ self.name=name
+
+
+[docs]
+ def__call__(self,fname,action,pooch):
+"""
+ Decompress the given file.
+
+ The output file will be either ``{fname}.decomp`` or the given *name*
+ class attribute.
+
+ Parameters
+ ----------
+ fname : str
+ Full path of the compressed file in local storage.
+ action : str
+ Indicates what action was taken by :meth:`pooch.Pooch.fetch` or
+ :func:`pooch.retrieve`:
+
+ - ``"download"``: File didn't exist locally and was downloaded
+ - ``"update"``: Local file was outdated and was re-download
+ - ``"fetch"``: File exists and is updated so it wasn't downloaded
+
+ pooch : :class:`pooch.Pooch`
+ The instance of :class:`pooch.Pooch` that is calling this.
+
+ Returns
+ -------
+ fname : str
+ The full path to the decompressed file.
+ """
+ ifself.nameisNone:
+ decompressed=fname+".decomp"
+ else:
+ decompressed=os.path.join(os.path.dirname(fname),self.name)
+ ifactionin("update","download")ornotos.path.exists(decompressed):
+ get_logger().info(
+ "Decompressing '%s' to '%s' using method '%s'.",
+ fname,
+ decompressed,
+ self.method,
+ )
+ module=self._compression_module(fname)
+ withopen(decompressed,"w+b")asoutput:
+ withmodule.open(fname)ascompressed:
+ shutil.copyfileobj(compressed,output)
+ returndecompressed
+
+
+ def_compression_module(self,fname):
+"""
+ Get the Python module compatible with fname and the chosen method.
+
+ If the *method* attribute is "auto", will select a method based on the
+ extension. If no recognized extension is in the file name, will raise a
+ ValueError.
+ """
+ error_archives="To unpack zip/tar archives, use pooch.Unzip/Untar instead."
+ ifself.methodnotinself.modules:
+ message=(
+ f"Invalid compression method '{self.method}'. "
+ f"Must be one of '{list(self.modules.keys())}'."
+ )
+ ifself.methodin{"zip","tar"}:
+ message=" ".join([message,error_archives])
+ raiseValueError(message)
+ ifself.method=="auto":
+ ext=os.path.splitext(fname)[-1]
+ ifextnotinself.extensions:
+ message=(
+ f"Unrecognized file extension '{ext}'. "
+ f"Must be one of '{list(self.extensions.keys())}'."
+ )
+ ifextin{".zip",".tar"}:
+ message=" ".join([message,error_archives])
+ raiseValueError(message)
+ returnself.modules[self.extensions[ext]]
+ returnself.modules[self.method]
+# Copyright (c) 2018 The Pooch Developers.
+# Distributed under the terms of the BSD 3-Clause License.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# This code is part of the Fatiando a Terra project (https://www.fatiando.org)
+#
+"""
+Misc utilities
+"""
+importlogging
+importos
+importtempfile
+importhashlib
+frompathlibimportPath
+fromurllib.parseimporturlsplit
+fromcontextlibimportcontextmanager
+importwarnings
+
+importplatformdirs
+frompackaging.versionimportVersion
+
+
+LOGGER=logging.Logger("pooch")
+LOGGER.addHandler(logging.StreamHandler())
+
+
+deffile_hash(*args,**kwargs):
+"""
+ WARNING: Importing this function from pooch.utils is DEPRECATED.
+ Please import from the top-level namespace (`from pooch import file_hash`)
+ instead, which is fully backwards compatible with pooch >= 0.1.
+
+ Examples
+ --------
+
+ >>> fname = "test-file-for-hash.txt"
+ >>> with open(fname, "w") as f:
+ ... __ = f.write("content of the file")
+ >>> print(file_hash(fname))
+ 0fc74468e6a9a829f103d069aeb2bb4f8646bad58bf146bb0e3379b759ec4a00
+ >>> import os
+ >>> os.remove(fname)
+
+ """
+ # pylint: disable=import-outside-toplevel
+ from.hashesimportfile_hashasnew_file_hash
+
+ message="""
+ Importing file_hash from pooch.utils is DEPRECATED. Please import from the
+ top-level namespace (`from pooch import file_hash`) instead, which is fully
+ backwards compatible with pooch >= 0.1.
+ """
+ warnings.warn(message,DeprecationWarning,stacklevel=2)
+ returnnew_file_hash(*args,**kwargs)
+
+
+
+[docs]
+defget_logger():
+r"""
+ Get the default event logger.
+
+ The logger records events like downloading files, unzipping archives, etc.
+ Use the method :meth:`logging.Logger.setLevel` of this object to adjust the
+ verbosity level from Pooch.
+
+ Returns
+ -------
+ logger : :class:`logging.Logger`
+ The logger object for Pooch
+ """
+ returnLOGGER
+
+
+
+
+[docs]
+defos_cache(project):
+r"""
+ Default cache location based on the operating system.
+
+ The folder locations are defined by the ``platformdirs`` package
+ using the ``user_cache_dir`` function.
+ Usually, the locations will be following (see the
+ `platformdirs documentation <https://platformdirs.readthedocs.io>`__):
+
+ * Mac: ``~/Library/Caches/<AppName>``
+ * Unix: ``~/.cache/<AppName>`` or the value of the ``XDG_CACHE_HOME``
+ environment variable, if defined.
+ * Windows: ``C:\Users\<user>\AppData\Local\<AppAuthor>\<AppName>\Cache``
+
+ Parameters
+ ----------
+ project : str
+ The project name.
+
+ Returns
+ -------
+ cache_path : :class:`pathlib.Path`
+ The default location for the data cache. User directories (``'~'``) are
+ not expanded.
+
+ """
+ returnPath(platformdirs.user_cache_dir(project))
+
+
+
+
+[docs]
+defcheck_version(version,fallback="master"):
+"""
+ Check if a version is PEP440 compliant and there are no unreleased changes.
+
+ For example, ``version = "0.1"`` will be returned as is but ``version =
+ "0.1+10.8dl8dh9"`` will return the fallback. This is the convention used by
+ `versioneer <https://github.com/warner/python-versioneer>`__ to mark that
+ this version is 10 commits ahead of the last release.
+
+ Parameters
+ ----------
+ version : str
+ A version string.
+ fallback : str
+ What to return if the version string has unreleased changes.
+
+ Returns
+ -------
+ version : str
+ If *version* is PEP440 compliant and there are unreleased changes, then
+ return *version*. Otherwise, return *fallback*.
+
+ Raises
+ ------
+ InvalidVersion
+ If *version* is not PEP440 compliant.
+
+ Examples
+ --------
+
+ >>> check_version("0.1")
+ '0.1'
+ >>> check_version("0.1a10")
+ '0.1a10'
+ >>> check_version("0.1+111.9hdg36")
+ 'master'
+ >>> check_version("0.1+111.9hdg36", fallback="dev")
+ 'dev'
+
+ """
+ parse=Version(version)
+ ifparse.localisnotNone:
+ returnfallback
+ returnversion
+
+
+
+defparse_url(url):
+"""
+ Parse a URL into 3 components:
+
+ <protocol>://<netloc>/<path>
+
+ Example URLs:
+
+ * http://127.0.0.1:8080/test.nc
+ * ftp://127.0.0.1:8080/test.nc
+ * doi:10.6084/m9.figshare.923450.v1/test.nc
+
+ The DOI is a special case. The protocol will be "doi", the netloc will be
+ the DOI, and the path is what comes after the last "/".
+ The only exception are Zenodo dois: the protocol will be "doi", the netloc
+ will be composed by the "prefix/suffix" and the path is what comes after
+ the second "/". This allows to support special cases of Zenodo dois where
+ the path contains forward slashes "/", created by the GitHub-Zenodo
+ integration service.
+
+ Parameters
+ ----------
+ url : str
+ The URL.
+
+ Returns
+ -------
+ parsed_url : dict
+ Three components of a URL (e.g.,
+ ``{'protocol':'http', 'netloc':'127.0.0.1:8080','path': '/test.nc'}``).
+
+ """
+ ifurl.startswith("doi://"):
+ raiseValueError(
+ f"Invalid DOI link '{url}'. You must not use '//' after 'doi:'."
+ )
+ ifurl.startswith("doi:"):
+ protocol="doi"
+ parts=url[4:].split("/")
+ if"zenodo"inparts[1].lower():
+ netloc="/".join(parts[:2])
+ path="/"+"/".join(parts[2:])
+ else:
+ netloc="/".join(parts[:-1])
+ path="/"+parts[-1]
+ else:
+ parsed_url=urlsplit(url)
+ protocol=parsed_url.schemeor"file"
+ netloc=parsed_url.netloc
+ path=parsed_url.path
+ return{"protocol":protocol,"netloc":netloc,"path":path}
+
+
+defcache_location(path,env=None,version=None):
+"""
+ Location of the cache given a base path and optional configuration.
+
+ Checks for the environment variable to overwrite the path of the local
+ cache. Optionally add *version* to the path if given.
+
+ Parameters
+ ----------
+ path : str, PathLike, list or tuple
+ The path to the local data storage folder. If this is a list or tuple,
+ we'll join the parts with the appropriate separator. Use
+ :func:`pooch.os_cache` for a sensible default.
+ version : str or None
+ The version string for your project. Will be appended to given path if
+ not None.
+ env : str or None
+ An environment variable that can be used to overwrite *path*. This
+ allows users to control where they want the data to be stored. We'll
+ append *version* to the end of this value as well.
+
+ Returns
+ -------
+ local_path : PathLike
+ The path to the local directory.
+
+ """
+ ifenvisnotNoneandenvinos.environandos.environ[env]:
+ path=os.environ[env]
+ ifisinstance(path,(list,tuple)):
+ path=os.path.join(*path)
+ ifversionisnotNone:
+ path=os.path.join(str(path),version)
+ path=os.path.expanduser(str(path))
+ returnPath(path)
+
+
+defmake_local_storage(path,env=None):
+"""
+ Create the local cache directory and make sure it's writable.
+
+ Parameters
+ ----------
+ path : str or PathLike
+ The path to the local data storage folder.
+ env : str or None
+ An environment variable that can be used to overwrite *path*. Only used
+ in the error message in case the folder is not writable.
+ """
+ path=str(path)
+ # Check that the data directory is writable
+ ifnotos.path.exists(path):
+ action="create"
+ else:
+ action="write to"
+
+ try:
+ ifaction=="create":
+ # When running in parallel, it's possible that multiple jobs will
+ # try to create the path at the same time. Use exist_ok to avoid
+ # raising an error.
+ os.makedirs(path,exist_ok=True)
+ else:
+ withtempfile.NamedTemporaryFile(dir=path):
+ pass
+ exceptPermissionErroraserror:
+ message=[
+ str(error),
+ f"| Pooch could not {action} data cache folder '{path}'.",
+ "Will not be able to download data files.",
+ ]
+ ifenvisnotNone:
+ message.append(
+ f"Use environment variable '{env}' to specify a different location."
+ )
+ raisePermissionError(" ".join(message))fromerror
+
+
+@contextmanager
+deftemporary_file(path=None):
+"""
+ Create a closed and named temporary file and make sure it's cleaned up.
+
+ Using :class:`tempfile.NamedTemporaryFile` will fail on Windows if trying
+ to open the file a second time (when passing its name to Pooch function,
+ for example). This context manager creates the file, closes it, yields the
+ file path, and makes sure it's deleted in the end.
+
+ Parameters
+ ----------
+ path : str or PathLike
+ The directory in which the temporary file will be created.
+
+ Yields
+ ------
+ fname : str
+ The path to the temporary file.
+
+ """
+ tmp=tempfile.NamedTemporaryFile(delete=False,dir=path)
+ # Close the temp file so that it can be opened elsewhere
+ tmp.close()
+ try:
+ yieldtmp.name
+ finally:
+ ifos.path.exists(tmp.name):
+ os.remove(tmp.name)
+
+
+defunique_file_name(url):
+"""
+ Create a unique file name based on the given URL.
+
+ The file name will be unique to the URL by prepending the name with the MD5
+ hash (hex digest) of the URL. The name will also include the last portion
+ of the URL.
+
+ The format will be: ``{md5}-{filename}.{ext}``
+
+ The file name will be cropped so that the entire name (including the hash)
+ is less than 255 characters long (the limit on most file systems).
+
+ Parameters
+ ----------
+ url : str
+ The URL with a file name at the end.
+
+ Returns
+ -------
+ fname : str
+ The file name, unique to this URL.
+
+ Examples
+ --------
+
+ >>> print(unique_file_name("https://www.some-server.org/2020/data.txt"))
+ 02ddee027ce5ebb3d7059fb23d210604-data.txt
+ >>> print(unique_file_name("https://www.some-server.org/2019/data.txt"))
+ 9780092867b497fca6fc87d8308f1025-data.txt
+ >>> print(unique_file_name("https://www.some-server.org/2020/data.txt.gz"))
+ 181a9d52e908219c2076f55145d6a344-data.txt.gz
+
+ """
+ md5=hashlib.md5(url.encode()).hexdigest()
+ fname=parse_url(url)["path"].split("/")[-1]
+ # Crop the start of the file name to fit 255 characters including the hash
+ # and the :
+ fname=fname[-(255-len(md5)-1):]
+ unique_name=f"{md5}-{fname}"
+ returnunique_name
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/v1.8.1/_sources/about.rst b/v1.8.1/_sources/about.rst
new file mode 100644
index 00000000..72d37def
--- /dev/null
+++ b/v1.8.1/_sources/about.rst
@@ -0,0 +1,57 @@
+.. _about:
+
+Why use Pooch?
+==============
+
+Use cases
+---------
+
+.. tab-set::
+
+ .. tab-item:: Just download a file
+
+ **Who**: Scientists/researchers/developers looking to simply download a
+ file.
+
+ Pooch makes it easy to download a file (one function call).
+ On top of that, it also comes with some bonus features:
+
+ * Download and cache your data files locally (so it's only downloaded
+ once).
+ * Make sure everyone running the code has the same version of the data
+ files by verifying cryptographic hashes.
+ * Multiple download protocols HTTP/FTP/SFTP and basic authentication.
+ * Download from Digital Object Identifiers (DOIs) issued by repositories
+ like figshare and Zenodo.
+ * Built-in utilities to unzip/decompress files upon download
+
+ **Start here:** :ref:`retrieve`
+
+ .. tab-item:: Manage sample data for a Python program
+
+ **Who**: Package developers wanting to include sample data for use in
+ tutorials and tests.
+
+ Pooch was designed for this! It offers:
+
+ * Pure Python and :ref:`minimal dependencies `.
+ * Download a file only if necessary.
+ * Verification of download integrity through cryptographic hashes.
+ * Extensible design: plug in custom download and post-processing functions.
+ * Built-in utilities to unzip/decompress files upon download
+ * Multiple download protocols HTTP/FTP/SFTP and basic authentication.
+ * User control of data cache location through environment variables.
+
+ **Start here:** :ref:`intermediate`
+
+History
+-------
+
+Pooch was born out of shared need between the
+`Fatiando a Terra `__ libraries and
+`MetPy `__.
+During the
+`Scipy Conference 2018 `__
+sprints, developers from both projects got together and, realizing the shared
+necessity, devised a package that would combine the best of the existing
+functionality already present in each project and extend it's capabilities.
diff --git a/v1.8.1/_sources/api/generated/pooch.DOIDownloader.rst b/v1.8.1/_sources/api/generated/pooch.DOIDownloader.rst
new file mode 100644
index 00000000..1e18a1d8
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.DOIDownloader.rst
@@ -0,0 +1,142 @@
+pooch.DOIDownloader
+===================
+
+.. currentmodule:: pooch
+
+.. autoclass:: DOIDownloader
+
+ .. rubric:: Methods Summary
+
+ .. autosummary::
+
+
+ DOIDownloader.__call__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+----
+
+
+
+.. automethod:: DOIDownloader.__call__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+.. raw:: html
+
+
\ No newline at end of file
diff --git a/v1.8.1/_sources/api/generated/pooch.Decompress.rst b/v1.8.1/_sources/api/generated/pooch.Decompress.rst
new file mode 100644
index 00000000..9283a594
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.Decompress.rst
@@ -0,0 +1,154 @@
+pooch.Decompress
+================
+
+.. currentmodule:: pooch
+
+.. autoclass:: Decompress
+
+ .. rubric:: Methods Summary
+
+ .. autosummary::
+
+
+ Decompress.__call__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+----
+
+
+
+.. automethod:: Decompress.__call__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+.. raw:: html
+
+
\ No newline at end of file
diff --git a/v1.8.1/_sources/api/generated/pooch.FTPDownloader.rst b/v1.8.1/_sources/api/generated/pooch.FTPDownloader.rst
new file mode 100644
index 00000000..06af1ea8
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.FTPDownloader.rst
@@ -0,0 +1,142 @@
+pooch.FTPDownloader
+===================
+
+.. currentmodule:: pooch
+
+.. autoclass:: FTPDownloader
+
+ .. rubric:: Methods Summary
+
+ .. autosummary::
+
+
+ FTPDownloader.__call__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+----
+
+
+
+.. automethod:: FTPDownloader.__call__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+.. raw:: html
+
+
\ No newline at end of file
diff --git a/v1.8.1/_sources/api/generated/pooch.HTTPDownloader.rst b/v1.8.1/_sources/api/generated/pooch.HTTPDownloader.rst
new file mode 100644
index 00000000..85bb9ab5
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.HTTPDownloader.rst
@@ -0,0 +1,142 @@
+pooch.HTTPDownloader
+====================
+
+.. currentmodule:: pooch
+
+.. autoclass:: HTTPDownloader
+
+ .. rubric:: Methods Summary
+
+ .. autosummary::
+
+
+ HTTPDownloader.__call__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+----
+
+
+
+.. automethod:: HTTPDownloader.__call__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+.. raw:: html
+
+
\ No newline at end of file
diff --git a/v1.8.1/_sources/api/generated/pooch.Pooch.rst b/v1.8.1/_sources/api/generated/pooch.Pooch.rst
new file mode 100644
index 00000000..ff9ca22c
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.Pooch.rst
@@ -0,0 +1,206 @@
+pooch.Pooch
+===========
+
+.. currentmodule:: pooch
+
+.. autoclass:: Pooch
+
+ .. rubric:: Methods Summary
+
+ .. autosummary::
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Pooch.fetch
+
+
+
+ Pooch.get_url
+
+
+
+ Pooch.is_available
+
+
+
+ Pooch.load_registry
+
+
+
+ Pooch.load_registry_from_doi
+
+
+
+----
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+.. automethod:: Pooch.fetch
+
+
+
+.. automethod:: Pooch.get_url
+
+
+
+.. automethod:: Pooch.is_available
+
+
+
+.. automethod:: Pooch.load_registry
+
+
+
+.. automethod:: Pooch.load_registry_from_doi
+
+
+
+.. raw:: html
+
+
\ No newline at end of file
diff --git a/v1.8.1/_sources/api/generated/pooch.SFTPDownloader.rst b/v1.8.1/_sources/api/generated/pooch.SFTPDownloader.rst
new file mode 100644
index 00000000..2bd161c4
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.SFTPDownloader.rst
@@ -0,0 +1,142 @@
+pooch.SFTPDownloader
+====================
+
+.. currentmodule:: pooch
+
+.. autoclass:: SFTPDownloader
+
+ .. rubric:: Methods Summary
+
+ .. autosummary::
+
+
+ SFTPDownloader.__call__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+----
+
+
+
+.. automethod:: SFTPDownloader.__call__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+.. raw:: html
+
+
\ No newline at end of file
diff --git a/v1.8.1/_sources/api/generated/pooch.Untar.rst b/v1.8.1/_sources/api/generated/pooch.Untar.rst
new file mode 100644
index 00000000..192b9de2
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.Untar.rst
@@ -0,0 +1,166 @@
+pooch.Untar
+===========
+
+.. currentmodule:: pooch
+
+.. autoclass:: Untar
+
+ .. rubric:: Methods Summary
+
+ .. autosummary::
+
+
+
+
+ Untar.__call__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+----
+
+
+
+
+
+.. automethod:: Untar.__call__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+.. raw:: html
+
+
\ No newline at end of file
diff --git a/v1.8.1/_sources/api/generated/pooch.Unzip.rst b/v1.8.1/_sources/api/generated/pooch.Unzip.rst
new file mode 100644
index 00000000..3f38a2aa
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.Unzip.rst
@@ -0,0 +1,170 @@
+pooch.Unzip
+===========
+
+.. currentmodule:: pooch
+
+.. autoclass:: Unzip
+
+ .. rubric:: Methods Summary
+
+ .. autosummary::
+
+
+
+
+
+
+ Unzip.__call__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+----
+
+
+
+
+
+
+
+.. automethod:: Unzip.__call__
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+.. raw:: html
+
+
\ No newline at end of file
diff --git a/v1.8.1/_sources/api/generated/pooch.check_version.rst b/v1.8.1/_sources/api/generated/pooch.check_version.rst
new file mode 100644
index 00000000..39784b0e
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.check_version.rst
@@ -0,0 +1,10 @@
+pooch.check\_version
+====================
+
+.. currentmodule:: pooch
+
+.. autofunction:: check_version
+
+.. raw:: html
+
+
diff --git a/v1.8.1/_sources/api/generated/pooch.create.rst b/v1.8.1/_sources/api/generated/pooch.create.rst
new file mode 100644
index 00000000..bc6ac8b6
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.create.rst
@@ -0,0 +1,10 @@
+pooch.create
+============
+
+.. currentmodule:: pooch
+
+.. autofunction:: create
+
+.. raw:: html
+
+
diff --git a/v1.8.1/_sources/api/generated/pooch.file_hash.rst b/v1.8.1/_sources/api/generated/pooch.file_hash.rst
new file mode 100644
index 00000000..bc9756bb
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.file_hash.rst
@@ -0,0 +1,10 @@
+pooch.file\_hash
+================
+
+.. currentmodule:: pooch
+
+.. autofunction:: file_hash
+
+.. raw:: html
+
+
diff --git a/v1.8.1/_sources/api/generated/pooch.get_logger.rst b/v1.8.1/_sources/api/generated/pooch.get_logger.rst
new file mode 100644
index 00000000..875eb92d
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.get_logger.rst
@@ -0,0 +1,10 @@
+pooch.get\_logger
+=================
+
+.. currentmodule:: pooch
+
+.. autofunction:: get_logger
+
+.. raw:: html
+
+
diff --git a/v1.8.1/_sources/api/generated/pooch.make_registry.rst b/v1.8.1/_sources/api/generated/pooch.make_registry.rst
new file mode 100644
index 00000000..41a65c5e
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.make_registry.rst
@@ -0,0 +1,10 @@
+pooch.make\_registry
+====================
+
+.. currentmodule:: pooch
+
+.. autofunction:: make_registry
+
+.. raw:: html
+
+
diff --git a/v1.8.1/_sources/api/generated/pooch.os_cache.rst b/v1.8.1/_sources/api/generated/pooch.os_cache.rst
new file mode 100644
index 00000000..42b51c95
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.os_cache.rst
@@ -0,0 +1,10 @@
+pooch.os\_cache
+===============
+
+.. currentmodule:: pooch
+
+.. autofunction:: os_cache
+
+.. raw:: html
+
+
diff --git a/v1.8.1/_sources/api/generated/pooch.retrieve.rst b/v1.8.1/_sources/api/generated/pooch.retrieve.rst
new file mode 100644
index 00000000..5c74c992
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.retrieve.rst
@@ -0,0 +1,10 @@
+pooch.retrieve
+==============
+
+.. currentmodule:: pooch
+
+.. autofunction:: retrieve
+
+.. raw:: html
+
+
diff --git a/v1.8.1/_sources/api/generated/pooch.test.rst b/v1.8.1/_sources/api/generated/pooch.test.rst
new file mode 100644
index 00000000..d96bfe65
--- /dev/null
+++ b/v1.8.1/_sources/api/generated/pooch.test.rst
@@ -0,0 +1,10 @@
+pooch.test
+==========
+
+.. currentmodule:: pooch
+
+.. autofunction:: test
+
+.. raw:: html
+
+
diff --git a/v1.8.1/_sources/api/index.rst b/v1.8.1/_sources/api/index.rst
new file mode 100644
index 00000000..a349c369
--- /dev/null
+++ b/v1.8.1/_sources/api/index.rst
@@ -0,0 +1,66 @@
+.. _api:
+
+List of functions and classes (API)
+===================================
+
+.. note::
+
+ **All functions and classes should be accessed from the** :mod:`pooch`
+ **top-level namespace.**
+
+ Modules inside of the :mod:`pooch` package are meant mostly for internal
+ organization. Please **avoid importing** directly from submodules since
+ functions/classes may be moved around.
+
+.. automodule:: pooch
+
+Core
+----
+
+.. autosummary::
+ :toctree: generated/
+
+ pooch.create
+ pooch.Pooch
+ pooch.retrieve
+
+Utilities
+---------
+
+.. autosummary::
+ :toctree: generated/
+
+ pooch.os_cache
+ pooch.make_registry
+ pooch.file_hash
+ pooch.check_version
+ pooch.get_logger
+
+Downloaders
+-----------
+
+.. autosummary::
+ :toctree: generated/
+
+ pooch.HTTPDownloader
+ pooch.FTPDownloader
+ pooch.SFTPDownloader
+ pooch.DOIDownloader
+
+Processors
+----------
+
+.. autosummary::
+ :toctree: generated/
+
+ pooch.Unzip
+ pooch.Untar
+ pooch.Decompress
+
+Miscellaneous
+-------------
+
+.. autosummary::
+ :toctree: generated/
+
+ pooch.test
diff --git a/v1.8.1/_sources/authentication.rst b/v1.8.1/_sources/authentication.rst
new file mode 100644
index 00000000..f92fb045
--- /dev/null
+++ b/v1.8.1/_sources/authentication.rst
@@ -0,0 +1,82 @@
+.. _authentication:
+
+Authentication
+==============
+
+HTTP authentication
+-------------------
+
+Use the :class:`~pooch.HTTPDownloader` class directly to provide login
+credentials to HTTP servers that require basic authentication. For example:
+
+.. code:: python
+
+ from pooch import HTTPDownloader
+
+
+ def fetch_protected_data():
+ """
+ Fetch a file from a server that requires authentication
+ """
+ # Let the downloader know the login credentials
+ download_auth = HTTPDownloader(auth=("my_username", "my_password"))
+ fname = GOODBOY.fetch("some-data.csv", downloader=download_auth)
+ data = pandas.read_csv(fname)
+ return data
+
+It's probably not a good idea to hard-code credentials in your code. One way
+around this is to ask users to set their own credentials through environment
+variables. The download code could look something like so:
+
+.. code:: python
+
+ import os
+
+
+ def fetch_protected_data():
+ """
+ Fetch a file from a server that requires authentication
+ """
+ # Get the credentials from the user's environment
+ username = os.environ.get("SOMESITE_USERNAME")
+ password = os.environ.get("SOMESITE_PASSWORD")
+ # Let the downloader know the login credentials
+ download_auth = HTTPDownloader(auth=(username, password))
+ fname = GOODBOY.fetch("some-data.csv", downloader=download_auth)
+ data = pandas.read_csv(fname)
+ return data
+
+
+FTP/SFTP with authentication
+----------------------------
+
+Pooch also comes with the :class:`~pooch.FTPDownloader` and
+:class:`~pooch.SFTPDownloader` downloaders that can be used
+when files are distributed over FTP or SFTP (secure FTP).
+
+.. note::
+
+ To download files over SFTP,
+ `paramiko `__ needs to be installed.
+
+
+Sometimes the FTP server doesn't support anonymous FTP and needs authentication
+or uses a non-default port.
+In these cases, pass in the downloader class explicitly (works with both FTP
+and SFTP):
+
+.. code:: python
+
+ import os
+
+
+ def fetch_c137():
+ """
+ Load the C-137 sample data as a pandas.DataFrame (over FTP this time).
+ """
+ username = os.environ.get("MYDATASERVER_USERNAME")
+ password = os.environ.get("MYDATASERVER_PASSWORD")
+ download_ftp = pooch.FTPDownloader(username=username, password=password)
+ fname = GOODBOY.fetch("c137.csv", downloader=download_ftp)
+ data = pandas.read_csv(fname)
+ return data
diff --git a/v1.8.1/_sources/changes.rst b/v1.8.1/_sources/changes.rst
new file mode 100644
index 00000000..d211f49b
--- /dev/null
+++ b/v1.8.1/_sources/changes.rst
@@ -0,0 +1,870 @@
+.. _changes:
+
+Changelog
+=========
+
+Version 1.8.1
+-------------
+
+Released on: 2024/02/19
+
+DOI: https://doi.org/10.5281/zenodo.10680982
+
+Bug fixes:
+
+* Use the ID instead of persistentID for Dataverse downloads since some repositories don't issue persistentIDs but all issue normal IDs (`#355 `__)
+* Ensure all archive members are unpacked in subsequent uses of ``Untar``/``Unzip`` if the first call only asked for a few members (`#365 `__)
+
+Documentation:
+
+* Move "Projects using Pooch" further up the README (`#386 `__)
+* Update the versions of sphinx and its plugins (`#385 `__)
+
+Maintenance:
+
+* Remove many deprecated pylint options (`#329 `__)
+* Use Dependabot to manage GitHub Actions (`#387 `__)
+* Simplify the test GitHub Actions workflow (`#384 `__)
+* Update format for Black 24.1.1 (`#383 `__)
+
+This release contains contributions from:
+
+* Mark Harfouche
+* Juan Nunez-Iglesias
+* Santiago Soler
+* Leonardo Uieda
+
+Version 1.8.0
+-------------
+
+Released on: 2023/10/24
+
+DOI: https://doi.org/10.5281/zenodo.10037888
+
+Bug fixes:
+
+* Fix bug: add support for old and new Zenodo APIs (`#375 `__)
+
+New features:
+
+* Only create local data directories if necessary (`#370 `__)
+* Speed up import time by lazy loading requests (`#328 `__)
+
+Maintenance:
+
+* Add support for Python 3.11 (`#348 `__)
+* Only run CI cron job for the upstream repository (`#361 `__)
+
+Documentation:
+
+* Add GemGIS to list of projects using Pooch (`#349 `__)
+* Fix spelling of Dataverse (`#353 `__)
+* Fix grammar on retrieve documentation (`#359 `__)
+
+This release contains contributions from:
+
+* Hugo van Kemenade
+* AlexanderJuestel
+* Mark Harfouche
+* Philip Durbin
+* Rob Luke
+* Santiago Soler
+* Stephan Hoyer
+
+
+Version 1.7.0
+-------------
+
+Released on: 2023/02/27
+
+DOI: https://doi.org/10.5281/zenodo.7678844
+
+Bug fixes:
+
+* Make archive extraction always take members into account (`#316 `__)
+* Figshare downloaders fetch the correct version, instead of always the latest one. (`#343 `__)
+
+New features:
+
+* Allow spaces in filenames in registry files (`#315 `__)
+* Refactor ``Pooch.is_available`` to use downloaders (`#322 `__)
+* Add support for downloading files from Dataverse DOIs (`#318 `__)
+* Add a new ``Pooch.load_registry_from_doi`` method that populates the Pooch registry using DOI-based data repositories (`#325 `__)
+* Support urls for Zenodo repositories created through the GitHub integration service, which include slashes in the filename of the main zip files (`#340 `__)
+* Automatically add a trailing slash to ``base_url`` on ``pooch.create`` (`#344 `__)
+
+Maintenance:
+
+* Drop support for Python 3.6 (`#299 `__)
+* Port from deprecated ``appdirs`` to ``platformdirs`` (`#339 `__)
+* Update version of Codecov's Action to v3 (`#345 `__)
+
+Documentation:
+
+* Update sphinx, theme, and sphinx-panels (`#300 `__)
+* Add CITATION.cff for the JOSS article (`#308 `__)
+* Use Markdown for the README (`#311 `__)
+* Improve docstring of `known_hash` in `retrieve` function (`#333 `__)
+* Replace link to Pooch's citation with a BibTeX code snippet (`#335 `__)
+
+Projects that started using Pooch:
+
+* Open AR-Sandbox (`#305 `__)
+* ``climlab`` (`#312 `__)
+* SciPy (`#320 `__)
+* ``napari`` (`#321 `__)
+* ``mne-python`` (`#323 `__)
+
+This release contains contributions from:
+
+* Alex Fikl
+* Anirudh Dagar
+* Björn Ludwig
+* Brian Rose
+* Dominic Kempf
+* Florian Wellmann
+* Gabriel Fu
+* Kyle I S Harrington
+* Leonardo Uieda
+* myd7349
+* Rowan Cockett
+* Santiago Soler
+
+Version 1.6.0
+-------------
+
+Released on: 2022/01/24
+
+DOI: https://doi.org/10.5281/zenodo.5793074
+
+.. warning::
+
+ **Pooch v1.6.0 is the last release that is compatible with Python 3.6.**
+
+Important notes:
+
+* Pooch now specifies version bounds for our required dependencies and a plan for dropping support for older versions. Please revise it if you depend on Pooch.
+
+Enhancements:
+
+* Add option to disable updates on hash mismatch (`#291 `__ and `#292 `__)
+* Allow enabling progress bars with an argument in ``Pooch.fetch`` and ``retrieve`` (`#277 `__)
+
+Documentation:
+
+* Use real data URLs in the README example code (`#295 `__)
+* Tell users to import from the top-level namespace (`#288 `__)
+* Update the contact link to `fatiando.org/contact `__ (`#282 `__)
+* Refer the community guides to `fatiando/community `__ (`#281 `__)
+* Mention in docs that figshare collections aren't supported (`#275 `__)
+
+Maintenance:
+
+* Replace Google Analytics for `Plausible `__ to make our docs more privacy-friendly (`#293 `__)
+* Use `Dependente `__ to capture dependencies on CI (`#289 `__)
+* Use ``build`` instead of setup.py (`#287 `__)
+* Run the tests weekly on GitHub Actions (`#286 `__)
+* Set minimum required version of dependencies (`#280 `__)
+* Rename "master" to "main" throughout the project (`#278 `__)
+* Remove trailing slash from GitHub handle in ``AUTHORS.md`` (`#279 `__)
+
+This release contains contributions from:
+
+* Santiago Soler
+* Genevieve Buckley
+* Ryan Abernathey
+* Ryan May
+* Leonardo Uieda
+
+Version 1.5.2
+-------------
+
+Released on: 2021/10/11
+
+DOI: https://doi.org/10.5281/zenodo.5560923
+
+Bug fixes:
+
+* Fix bug when unpacking an entire subfolder from an archive. Now both unpacking processors (``Untar`` and ``Unzip``) handle ``members`` that are folders (not files) correctly. (`#266 `__)
+
+Enhancements:
+
+* Add support for Python 3.10 (`#260 `__)
+* Point to the user's code for the file_hash warning instead of our internal code (which isn't very useful) (`#259 `__)
+
+Documentation:
+
+* Fix typo in a variable name of the examples in the documentation (`#268 `__)
+* Fix typo when specifying the SFTP protocol in the about page (`#267 `__)
+
+Maintenance:
+
+* Remove old testing checks if running on TravisCI (`#265 `__)
+
+This release contains contributions from:
+
+* Santiago Soler
+* Hugo van Kemenade
+* Mark Harfouche
+* Leonardo Uieda
+
+Version 1.5.1
+-------------
+
+Released on: 2021/08/24
+
+DOI: https://doi.org/10.5281/zenodo.5242882
+
+.. warning::
+
+ **Please use** ``from pooch import file_hash`` **instead of** ``from
+ pooch.utils import file_hash``. This is backwards compatible with all
+ previous versions of Pooch. We recommend importing all functions and
+ classes from the top-level namespace.
+
+Bug fixes:
+
+* Make ``file_hash`` accessible from the ``pooch.utils`` module again. Moving
+ this function to ``pooch.hashes`` caused crashes downstream. To prevent these
+ crashes, add a wrapper back to utils that issues a warning that users should
+ import from the top-level namespace instead.
+ (`#257 `__)
+* Use a mirror of the test data directory in tests that write to it.
+ (`#255 `__)
+* Add a pytest mark for tests accessing the network so that they can easily
+ excluded when testing offline. (`#254 `__)
+
+This release contains contributions from:
+
+* Antonio Valentino
+* Leonardo Uieda
+
+Version 1.5.0
+-------------
+
+Released on: 2021/08/23
+
+DOI: https://doi.org/10.5281/zenodo.5235242
+
+New features:
+
+* Add support for non-cryptographic hashes from the xxhash package. They aren't
+ as safe (but safe enough) and compute in fractions of the time from SHA or
+ MD5. This makes it feasible to use hash checking on large datasets. (`#242
+ `__)
+* Add support for using figshare and Zenodo DOIs as URLs (with the protocol
+ ``doi:{DOI}/{file name}``, which works out-of-the-box with ``Pooch.fetch``
+ and ``retrieve``). Can only download 1 file from the archive (not the full
+ archive) and the file name must be specified in the URL. (`#241
+ `__)
+
+Maintenance:
+
+* Move hash functions to their own private module. No changes to the public
+ API. (`#244 `__)
+* Run CI jobs on Python version extremes instead of all supported versions
+ (`#243 `__)
+
+This release contains contributions from:
+
+* Mark Harfouche
+* Leonardo Uieda
+
+Version 1.4.0
+-------------
+
+Released on: 2021/06/08
+
+DOI: https://doi.org/10.5281/zenodo.4914758
+
+Bug fixes:
+
+* Fix bug in ``Untar`` and ``Unzip`` when the archive contains subfolders
+ (`#224 `__)
+
+Documentation:
+
+* New theme (``sphinx-book-theme``) and layout of the documentation (`#236
+ `__ `#237
+ `__ `#238
+ `__)
+
+Enhancements:
+
+* Add support for non-tqdm progress bars on HTTPDownloader (`#228
+ `__)
+* Allow custom unpack locations in ``Untar`` and ``Unzip`` (`#224
+ `__)
+
+Maintenance:
+
+* Replace versioneer with setuptools-scm (`#235
+ `__)
+* Automatically check license notice on code files (`#231
+ `__)
+* Don't store documentation HTML as CI build artifacts (`#221
+ `__)
+
+This release contains contributions from:
+
+* Leonardo Uieda
+* Agustina Pesce
+* Clément Robert
+* Daniel McCloy
+
+Version 1.3.0
+-------------
+
+Released on: 2020/11/27
+
+DOI: https://doi.org/10.5281/zenodo.4293216
+
+Bug fixes:
+
+* Properly handle capitalized hashes. On Windows, users might sometimes get
+ capitalized hashes from the system. To avoid false hash mismatches, convert
+ stored and computed hashes to lowercase before doing comparisons. Convert
+ hashes to lowercase when reading from the registry to make sure stored hashes
+ are always lowercase. (`#214 `__)
+
+New features:
+
+* Add option to retry downloads if they fail. The new ``retry_if_failed``
+ option to ``pooch.create`` and ``pooch.Pooch`` allows retrying the download
+ the specified number of times in case of failures due to hash mismatches
+ (coming from Pooch) or network issues (coming from ``requests``). This is
+ useful for running downloads on CI that tend to fail sporadically. Waits a
+ period of time between consecutive downloads starting with 1s and increasing
+ up to 10s in 1s increments. (`#215
+ `__)
+* Allow user defined decompressed file names. Introduce new ``name`` argument
+ to ``pooch.Decompress`` to allow user defined file names. Defaults to the
+ previous naming convention for backward compatibility. (`#203
+ `__)
+
+Documentation:
+
+* Add seaborn-image to list of packages using Pooch (`#218
+ `__)
+
+Maintenance:
+
+* Add support for Python 3.9. (`#220
+ `__)
+* Drop support for Python 3.5. (`#204
+ `__)
+* Use pip instead of conda to speed up Actions (`#216
+ `__)
+* Add license and copyright notice to every .py file (`#213
+ `__)
+
+This release contains contributions from:
+
+* Leonardo Uieda
+* Danilo Horta
+* Hugo van Kemenade
+* SarthakJariwala
+
+
+Version 1.2.0
+-------------
+
+Released on: 2020/09/10
+
+DOI: https://doi.org/10.5281/zenodo.4022246
+
+.. warning::
+
+ **Pooch v1.2.0 is the last release that is compatible with Python 3.5.**
+
+Bug fixes:
+
+* Fix FTP availability check when the file is in a directory. If the data file
+ is not in the base directory, the ``Pooch.is_available`` test was broken
+ since we were checking for the full path in ``ftp.nlst`` instead of just the
+ file name. (`#191 `__)
+
+New features:
+
+* Add the SFTPDownloader class for secure FTP downloads (`#165
+ `__)
+* Expose Pooch version as ``pooch.__version__`` (`#179
+ `__)
+* Allow line comments in registry files with ``#`` (`#180
+ `__)
+
+Enhancements:
+
+* Point to Unzip/tar from Decompress docs and errors (`#200
+ `__)
+
+Documentation:
+
+* Re-factor the documentation into separate pages (`#202
+ `__)
+* Add warning to the docs about dropping Python 3.5 (`#201
+ `__)
+* Add `histolab `__ to the Pooch-powered
+ projects (`#189 `__)
+
+Maintenance:
+
+* Push documentation to GitHub Pages using Actions (`#198
+ `__)
+* Add GitHub Actions workflow for publishing to PyPI (`#196
+ `__)
+* Set up GitHub Actions for testing and linting (`#194
+ `__)
+* Test FTP downloads using a local test server (`#192
+ `__)
+
+This release contains contributions from:
+
+* Leonardo Uieda
+* Hugo van Kemenade
+* Alessia Marcolini
+* Luke Gregor
+* Mathias Hauser
+
+Version 1.1.1
+-------------
+
+Released on: 2020/05/14
+
+DOI: https://doi.org/10.5281/zenodo.3826458
+
+Bug fixes:
+
+* Delay data cache folder creation until the first download is attempted. As
+ seen in `recent issues in scikit-image
+ `__, creating the
+ data folder in ``pooch.create`` can cause problems since this function is
+ called at import time. This means that importing the package in parallel can
+ cause race conditions and crashes. To prevent that from happening, delay the
+ creation of the cache folder until ``Pooch.fetch`` or ``retrieve`` are
+ called.
+ (`#173 `__)
+* Allow the data folder to already exist when creating it. This is can help
+ cope with parallel execution as well.
+ (`#171 `__)
+
+Documentation:
+
+* Added scikit-image to list of Pooch users.
+ (`#168 `__)
+* Fix typo in README and front page contributing section.
+ (`#166 `__)
+
+This release contains contributions from:
+
+* Leonardo Uieda
+* Egor Panfilov
+* Rowan Cockett
+
+Version 1.1.0
+-------------
+
+Released on: 2020/04/13
+
+DOI: https://doi.org/10.5281/zenodo.3747184
+
+New features:
+
+* New function ``pooch.retrieve`` to fetch single files This is much more
+ convenient than setting up a ``Pooch`` while retaining the hash checks and
+ use of downloaders and processors. It automatically selects a unique file
+ name and saves files to a cache folder.
+ (`#152 `__)
+* Allow to use of different hashing algorithms (other than SHA256). Optionally
+ specify the hash as ``alg:hash`` and allow ``pooch.Pooch`` to recognize the
+ algorithm when comparing hashes. Setting an algorithsm is optional and
+ omiting it defaults to SHA256. This is particularly useful when data are
+ coming from external sources and published hashes are already available.
+ (`#133 `__)
+
+Documentation:
+
+* Add example for fetching datasets that change on the server, for which the
+ hash check would always fail.
+ (`#144 `__)
+* Fix path examples in docstring of ``pooch.os_cache``. The docstring mentioned
+ the data path as examples instead of the cache path.
+ (`#140 `__)
+* Add example of creating a registry when you don't have the data files locally
+ and would have to download them manually. The example uses the
+ ``pooch.retrieve`` function to automate the process. The example covers two
+ cases: when all remote files share the same base URL and when every file has
+ its own URL.
+ (`#161 `__)
+
+Maintenance:
+
+* A lot of general refactoring of the internals of Pooch to facilitate
+ development of the new ``pooch.retrieve`` function
+ (`#159 `__
+ `#157 `__
+ `#156 `__
+ `#151 `__
+ `#149 `__)
+
+This release contains contributions from:
+
+* Leonardo Uieda
+* Santiago Soler
+* Kacper Kowalik
+* Lucas Martin-King
+* Zac Flamig
+
+Version 1.0.0
+-------------
+
+Released on: 2020/01/28
+
+DOI: https://doi.org/10.5281/zenodo.3629329
+
+This release marks the stabilization of the Pooch API. Further changes to the
+1.* line will be fully backwards compatible (meaning that updating Pooch should
+not break existing code). If there is great need to make backwards incompatible
+changes, we will release a 2.* line. In that case, bug fixes will still be
+ported to the 1.* line for a period of time.
+
+Improvements:
+
+* Allow blank lines in registry files. Previously, they would cause an error.
+ (`#138 `__)
+
+**Backwards incompatible changes**:
+
+* Using Python's ``logging`` module to instead of ``warnings`` to inform users
+ of download, update, and decompression/unpacking actions. This allows
+ messages to be logged with different priorities and the user filter out log
+ messages or silence Pooch entirely. Introduces the function
+ ``pooch.get_logger`` to access the ``logging`` object used by Pooch. **Users
+ who relied on Pooch issuing warnings will need to update to capturing logs
+ instead.** All other parts of the API remain unchanged.
+ (`#115 `__)
+
+This release contains contributions from:
+
+* Daniel Shapero
+
+Version 0.7.2
+-------------
+
+Released on: 2020/01/17
+
+🚨 **Announcement:** 🚨
+We now have a `JOSS paper about Pooch `__!
+Please :ref:`cite it ` when you use Pooch for your research.
+(`#116 `__ with reviews in
+`#132 `__ and
+`#134 `__)
+
+This is minor release which only updates the citation information to
+the new JOSS paper. No DOI was issued for this release since there are
+no code or documentation changes.
+
+Version 0.7.1
+-------------
+
+Released on: 2020/01/17
+
+DOI: https://doi.org/10.5281/zenodo.3611376
+
+Improvements:
+
+* Better error messages when hashes don't match. Include the file name in the
+ exception for a hash mismatch between a downloaded file and the registry.
+ Before, we included the name of temporary file, which wasn't very
+ informative.
+ (`#128 `__)
+* Better error message for malformed registry files. When loading a registry
+ file, inform the name of the file and include the offending content in the
+ error message instead of just the line number.
+ (`#129 `__)
+
+Maintenance:
+
+* Change development status flag in ``setup.py`` to "stable" instead of
+ "alpha".
+ (`#127 `__)
+
+This release was reviewed at the `Journal of Open Source Software
+`__. The code and
+software paper contain contributions from:
+
+* Anderson Banihirwe
+* Martin Durant
+* Mark Harfouche
+* Hugo van Kemenade
+* John Leeman
+* Rémi Rampin
+* Daniel Shapero
+* Santiago Rubén Soler
+* Matthew Turk
+* Leonardo Uieda
+
+Version 0.7.0
+-------------
+
+Released on: 2019/11/19
+
+DOI: https://doi.org/10.5281/zenodo.3547640
+
+New features:
+
+* New ``pooch.FTPDownloader`` class for downloading files over FTP. Uses the
+ standard library ``ftplib``. The appropriate downloader is automatically
+ selected by ``pooch.Pooch.fetch`` based on the URL (for anonymous FTP only),
+ so no configuration is required.
+ If authentication is required, ``pooch.FTPDownloader`` provides the need
+ support. Ported from
+ `NCAR/aletheia-data `__ by the author.
+ (`#118 `__)
+* Support for file-like objects to ``Pooch.load_registry`` (opened either in
+ binary or text mode).
+ (`#117 `__)
+
+Maintenance:
+
+* Testing and official support for Python 3.8.
+ (`#113 `__)
+* 🚨 **Drop support for Python 2.7.** 🚨 Remove conditional dependencies and CI
+ jobs.
+ (`#100 `__)
+
+Documentation:
+
+* In the tutorial, use ``pkg_resources.resource_stream()`` from setuptools to
+ load the ``registry.txt`` file. It's less error-prone than using ``os.path``
+ and ``__file__`` and allows the package to work from zip files.
+ (`#120 `__)
+* Docstrings formatted to 79 characters (instead of 88) for better rendering in
+ Jupyter notebooks and IPython. These displays are limited to 80 chars so the
+ longer lines made the docstring unreadable.
+ (`#123 `__)
+
+This release contains contributions from:
+
+* Anderson Banihirwe
+* Hugo van Kemenade
+* Remi Rampin
+* Leonardo Uieda
+
+Version 0.6.0
+-------------
+
+Released on: 2019/10/22
+
+DOI: https://doi.org/10.5281/zenodo.3515031
+
+🚨 **Pooch v0.6.0 is the last release to support Python 2.7** 🚨
+
+New features:
+
+* Add optional download progress bar to ``pooch.HTTPDownloader``
+ (`#97 `__)
+
+Maintenance:
+
+* Warn that 0.6.0 is the last version to support Python 2.7
+ (`#108 `__)
+
+Documentation:
+
+* Update contact information to point to our Slack channel
+ (`#107 `__)
+* Add icepack to list of projects using Pooch
+ (`#98 `__)
+
+This release contains contributions from:
+
+* Daniel Shapero
+* Leonardo Uieda
+
+Version 0.5.2
+-------------
+
+Released on: 2019/06/24
+
+Maintenance:
+
+* Add back support for Python 3.5 with continuous integration tests. No code changes
+ were needed, only removing the restriction from ``setup.py``.
+ (`#93 `__)
+
+This release contains contributions from:
+
+* Leonardo Uieda
+
+Version 0.5.1
+-------------
+
+Released on: 2019/05/21
+
+Documentation fixes:
+
+* Fix formatting error in ``pooch.Decompress`` docstring.
+ (`#81 `__)
+* Fix wrong imports in the usage guide for post-processing hooks.
+ (`#84 `__)
+* Add section to the usage guide explaining when to use ``pooch.Decompress``.
+ (`#85 `__)
+
+This release contains contributions from:
+
+* Santiago Soler
+* Leonardo Uieda
+
+Version 0.5.0
+-------------
+
+Released on: 2019/05/20
+
+New features:
+
+* New processor ``pooch.Decompress`` saves a decompressed version of the downloaded
+ file. Supports gzip, lzma/xz, and bzip2 compression. **Note**: Under Python 2.7, lzma
+ and bzip2 require the ``backports.lzma`` and ``bz2file`` packages as well. These are
+ soft dependencies and not required to use Pooch. See :ref:`install`. (`#78
+ `__)
+* New processor ``pooch.Untar`` unpacks files contained in a downloaded tar archive
+ (with or without compression). (`#77 `__)
+
+This release contains contributions from:
+
+* Matthew Turk
+* Leonardo Uieda
+
+Version 0.4.0
+-------------
+
+Released on: 2019/05/01
+
+New features:
+
+* Add customizable downloaders. Delegate file download into separate classes that can be
+ passed to ``Pooch.fetch``. Created the ``HTTPDownloader`` class (used by default)
+ which can also be used to download files that require authentication/login. (`#66
+ `__)
+* Add post-download processor hooks to ``Pooch.fetch``. Allows users to pass in a
+ function that is executed right before returning and can overwrite the file path that
+ is returned by ``fetch``. Use this, for example, to perform unpacking/decompression
+ operations on larger files that can be time consuming and we only want to do once.
+ (`#59 `__)
+* Add the ``Unzip`` post-download processor to extract files from a downloaded zip
+ archive. Unpacks files into a directory in the local store and returns a list of all
+ unzipped files. (`#72 `__)
+* Make the ``check_version`` function public. It's used internally but will be useful in
+ examples that want to download things from the pooch repository. (`#69
+ `__)
+
+Maintenance:
+
+* Pin sphinx to version 1.8.5. New versions of Sphinx (2.0.*) are messing up the
+ numpydoc style docstrings. (`#64 `__)
+
+This release contains contributions from:
+
+* Santiago Soler
+* Leonardo Uieda
+
+Version 0.3.1
+-------------
+
+Released on: 2019/03/28
+
+Minor patches:
+
+* Add a project logo (`#57 `__)
+* Replace ``http`` with ``https`` in the ``README.rst`` to avoid mixed content warnings
+ in some browsers (`#56 `__)
+
+Version 0.3.0
+-------------
+
+Released on: 2019/03/27
+
+New features:
+
+* Use the ``appdirs`` library to get the cache directory. **Could change the default
+ data location on all platforms**. Locations are compatible with the
+ `XDG Base Directory Specification `__
+ (`#45 `__)
+* Add method ``Pooch.is_available`` to check remote file availability
+ (`#50 `__)
+* Add ``Pooch.registry_files`` property to get a name of all files in the registry
+ (`#42 `__)
+* Make ``Pooch.get_url`` a public method to get the download URL for a given file
+ (`#55 `__)
+
+Maintenance:
+
+* **Drop support for Python 3.5**. Pooch now requires Python >= 3.6.
+ (`#52 `__)
+* Add a private method to check if a file is in the registry (`#49 `__)
+* Fix typo in the ``Pooch.load_registry`` docstring (`#41 `__)
+
+This release contains contributions from:
+
+* Santiago Soler
+* Rémi Rampin
+* Leonardo Uieda
+
+Version 0.2.1
+-------------
+
+Released on: 2018/11/15
+
+Bug fixes:
+
+* Fix unwanted ``~`` directory creation when not using a ``version`` in ``pooch.create``
+ (`#37 `__)
+
+
+Version 0.2.0
+-------------
+
+Released on: 2018/10/31
+
+Bug fixes:
+
+* Avoid copying of files across the file system (`#33 `__)
+* Correctly delete temporary downloads on error (`#32 `__)
+
+New features:
+
+* Allow custom download URLs for individual files (`#30 `__)
+* Allow dataset versioning to be optional (`#29 `__)
+
+Maintenance:
+
+* Move URLs building to a dedicated method for easy subclassing (`#31 `__)
+* Add testing and support for Python 3.7 (`#25 `__)
+
+
+Version 0.1.1
+-------------
+
+Released on: 2018/08/30
+
+Bug fixes:
+
+* Check if the local data folder is writable and warn the user instead of crashing
+ (`#23 `__)
+
+
+Version 0.1
+-----------
+
+Released on: 2018/08/20
+
+* Fist release of Pooch. Manages downloading sample data files over HTTP from a server
+ and storing them in a local directory. Main features:
+
+ - Download a file only if it's not in the local storage.
+ - Check the SHA256 hash to make sure the file is not corrupted or needs updating.
+ - If the hash is different from the registry, Pooch will download a new version of
+ the file.
+ - If the hash still doesn't match, Pooch will raise an exception warning of possible
+ data corruption.
diff --git a/v1.8.1/_sources/citing.rst b/v1.8.1/_sources/citing.rst
new file mode 100644
index 00000000..9c76d397
--- /dev/null
+++ b/v1.8.1/_sources/citing.rst
@@ -0,0 +1,3 @@
+.. _citing:
+
+.. include:: ../CITATION.rst
diff --git a/v1.8.1/_sources/compatibility.rst b/v1.8.1/_sources/compatibility.rst
new file mode 100644
index 00000000..ab9e3371
--- /dev/null
+++ b/v1.8.1/_sources/compatibility.rst
@@ -0,0 +1,71 @@
+.. _compatibility:
+
+Version compatibility
+=====================
+
+Pooch backwards incompatible changes
+------------------------------------
+
+We try to retain backwards compatibility whenever possible. Major breaking
+changes to the Pooch API will be marked by a major release and deprecation
+warnings will be issued in previous releases to give developers ample time to
+adapt.
+
+If there are any backwards incompatible changes, they will be listed below:
+
+.. list-table::
+ :widths: 20 10 70
+
+ * - **Version introduced**
+ - **Severity**
+ - **Notes**
+ * - v1.0.0
+ - Low
+ - We replaced use of ``warning`` with the ``logging`` module for all
+ messages issued by Pooch. This allows messages to be logged with
+ different priorities and the user filter out log messages or silence
+ Pooch entirely. **Users who relied on Pooch issuing warnings will need
+ to update to capturing logs instead.** The vast majority of users are
+ unaffected.
+
+.. _dependency-versions:
+
+Supported dependency versions
+-----------------------------
+
+Pooch follows the recommendations in
+`NEP29 `__ for setting
+the minimum required version of our dependencies.
+In short, we support **all minor releases of our dependencies from the previous
+24 months** before a Pooch release with a minimum of 2 minor releases.
+
+We follow this guidance conservatively and won't require newer versions if the
+older ones are still working without causing problems.
+Whenever support for a version is dropped, we will include a note in the
+:ref:`changes`.
+
+.. note::
+
+ This was introduced in Pooch v1.6.0.
+
+
+.. _python-versions:
+
+Supported Python versions
+-------------------------
+
+If you require support for older Python versions, please pin Pooch to the
+following releases to ensure compatibility:
+
+.. list-table::
+ :widths: 40 60
+
+ * - **Python version**
+ - **Last compatible Pooch release**
+ * - 2.7
+ - 0.6.0
+ * - 3.5
+ - 1.2.0
+ * - 3.6
+ - 1.6.0
+
diff --git a/v1.8.1/_sources/decompressing.rst b/v1.8.1/_sources/decompressing.rst
new file mode 100644
index 00000000..e9f4ae86
--- /dev/null
+++ b/v1.8.1/_sources/decompressing.rst
@@ -0,0 +1,56 @@
+.. _decompressing:
+
+Decompressing
+=============
+
+If you have a compressed file that is not an archive (zip or tar), you can use
+:class:`pooch.Decompress` to decompress it after download.
+
+For example, large binary files can be compressed with ``gzip`` to reduce
+download times but will need to be decompressed before loading, which can be
+slow.
+You can trade storage space for speed by keeping a decompressed copy of the
+file:
+
+.. code:: python
+
+ from pooch import Decompress
+
+ def fetch_compressed_file():
+ """
+ Load a large binary file that has been gzip compressed.
+ """
+ # Pass in the processor to decompress the file on download
+ fname = GOODBOY.fetch("large-binary-file.npy.gz", processor=Decompress())
+ # The file returned is the decompressed version which can be loaded by
+ # numpy
+ data = numpy.load(fname)
+ return data
+
+:class:`pooch.Decompress` returns ``"large-binary-file.npy.gz.decomp"`` as the
+decompressed file name by default.
+You can change this behaviour by passing a file name instead:
+
+.. code:: python
+
+ import os
+ from pooch import Decompress
+
+ def fetch_compressed_file():
+ """
+ Load a large binary file that has been gzip compressed.
+ """
+ # Pass in the processor to decompress the file on download
+ fname = GOODBOY.fetch("large-binary-file.npy.gz",
+ processor=Decompress(name="a-different-file-name.npy"),
+ )
+ # The file returned is now named "a-different-file-name.npy"
+ data = numpy.load(fname)
+ return data
+
+.. warning::
+
+ Passing in ``name`` can cause existing data to be lost!
+ For example, if a file already exists with the specified name it will be
+ overwritten with the new decompressed file content.
+ **Use this option with caution.**
diff --git a/v1.8.1/_sources/downloaders.rst b/v1.8.1/_sources/downloaders.rst
new file mode 100644
index 00000000..349fb907
--- /dev/null
+++ b/v1.8.1/_sources/downloaders.rst
@@ -0,0 +1,126 @@
+.. _downloaders:
+
+Downloaders: Customizing the download
+=====================================
+
+By default, :meth:`pooch.Pooch.fetch` and :meth:`pooch.retrieve` will detect
+the download protocol from the given URL (HTTP, FTP, SFTP, DOI) and use the
+appropriate download method.
+Sometimes this is not enough: some servers require logins, redirections, or
+other non-standard operations.
+To get around this, use the ``downloader`` argument of
+:meth:`~pooch.Pooch.fetch` and :meth:`~pooch.retrieve`.
+
+Downloaders are Python *callable objects* (like functions or classes with a
+``__call__`` method) and must have the following format:
+
+.. code:: python
+
+ def mydownloader(url, output_file, pooch):
+ '''
+ Download a file from the given URL to the given local file.
+
+ The function **must** take the following arguments (in order).
+
+ Parameters
+ ----------
+ url : str
+ The URL to the file you want to download.
+ output_file : str or file-like object
+ Path (and file name) to which the file will be downloaded.
+ pooch : pooch.Pooch
+ The instance of the Pooch class that is calling this function.
+
+ No return value is required.
+ '''
+ ...
+
+Pooch provides downloaders for HTTP, FTP, and SFTP that support authentication
+and optionally printing progress bars.
+See :ref:`api` for a list of available downloaders.
+
+Common uses of downloaders include:
+
+* Passing :ref:`login credentials ` to HTTP and FTP servers
+* Printing :ref:`progress bars `
+
+
+Creating your own downloaders
+-----------------------------
+
+If your use case is not covered by our downloaders, you can implement your own.
+:meth:`pooch.Pooch.fetch` and :func:`pooch.retrieve` will accept any *callable
+obejct* that has the signature specified above. As an example, consider the
+case in which the login credentials need to be provided to a site that is
+redirected from the original download URL:
+
+.. code:: python
+
+ import requests
+
+
+ def redirect_downloader(url, output_file, pooch):
+ """
+ Download after following a redirection.
+ """
+ # Get the credentials from the user's environment
+ username = os.environ.get("SOMESITE_USERNAME")
+ password = os.environ.get("SOMESITE_PASSWORD")
+ # Make a request that will redirect to the login page
+ login = requests.get(url)
+ # Provide the credentials and download from the new URL
+ download = HTTPDownloader(auth=(username, password))
+ download(login.url, output_file, mypooch)
+
+
+ def fetch_protected_data():
+ """
+ Fetch a file from a server that requires authentication
+ """
+ fname = GOODBOY.fetch("some-data.csv", downloader=redirect_downloader)
+ data = pandas.read_csv(fname)
+ return data
+
+
+Availability checks
+-------------------
+
+**Optionally**, downloaders can take a ``check_only`` keyword argument (default
+to ``False``) that makes them only check if a given file is available for
+download **without** downloading the file.
+This makes a downloader compatible with :meth:`pooch.Pooch.is_available`.
+
+In this case, the downloader should return a boolean:
+
+.. code:: python
+
+ def mydownloader(url, output_file, pooch, check_only=False):
+ '''
+ Download a file from the given URL to the given local file.
+
+ The function **must** take the following arguments (in order).
+
+ Parameters
+ ----------
+ url : str
+ The URL to the file you want to download.
+ output_file : str or file-like object
+ Path (and file name) to which the file will be downloaded.
+ pooch : pooch.Pooch
+ The instance of the Pooch class that is calling this function.
+ check_only : bool
+ If True, will only check if a file exists on the server and
+ **without downloading the file**. Will return ``True`` if the file
+ exists and ``False`` otherwise.
+
+ Returns
+ -------
+ None or availability
+ If ``check_only==True``, returns a boolean indicating if the file
+ is available on the server. Otherwise, returns ``None``.
+ '''
+ ...
+
+If a downloader does not implement an availability check (i.e., doesn't take
+``check_only`` as a keyword argument), then :meth:`pooch.Pooch.is_available`
+will raise a ``NotImplementedError``.
diff --git a/v1.8.1/_sources/hashes.rst b/v1.8.1/_sources/hashes.rst
new file mode 100644
index 00000000..ba30f2d0
--- /dev/null
+++ b/v1.8.1/_sources/hashes.rst
@@ -0,0 +1,131 @@
+.. _hashes:
+
+Hashes: Calculating and bypassing
+=================================
+
+Pooch uses hashes to check if files are up-to-date or possibly
+corrupted:
+
+* If a file exists in the local folder, Pooch will check that its hash matches
+ the one in the registry. If it doesn't, we'll assume that it needs to be
+ updated.
+* If a file needs to be updated or doesn't exist, Pooch will download it from
+ the remote source and check the hash. If the hash doesn't match, an exception
+ is raised to warn of possible file corruption.
+* Cryptographic hashes may be used where users wish to ensure the security of
+ their download.
+
+Calculating hashes
+------------------
+
+You can generate hashes for your data files using ``openssl`` in the terminal:
+
+.. code:: bash
+
+ $ openssl sha256 data/c137.csv
+ SHA256(data/c137.csv)= baee0894dba14b12085eacb204284b97e362f4f3e5a5807693cc90ef415c1b2d
+
+Or using the :func:`pooch.file_hash` function (which is a convenient way of
+calling Python's :mod:`hashlib`):
+
+.. code:: python
+
+ import pooch
+ print(pooch.file_hash("data/c137.csv"))
+
+
+Specifying the hash algorithm
+-----------------------------
+
+By default, Pooch uses `SHA256 `__
+hashes.
+Other hash methods that are available in :mod:`hashlib` can also be used:
+
+.. code:: python
+
+ import pooch
+ print(pooch.file_hash("data/c137.csv", alg="sha512"))
+
+In this case, you can specify the hash algorithm in the **registry** by
+prepending it to the hash, for example ``"md5:0hljc7298ndo2"`` or
+``"sha512:803o3uh2pecb2p3829d1bwouh9d"``.
+Pooch will understand this and use the appropriate method.
+
+
+Bypassing the hash check
+------------------------
+
+Sometimes we might not know the hash of the file or it could change on the
+server periodically.
+To bypass the check, we can set the hash value to ``None`` when specifying the
+``registry`` argument for :func:`pooch.create`
+(or the ``known_hash`` in :func:`pooch.retrieve`).
+
+In this example, we want to use Pooch to download a list of weather stations
+around Australia:
+
+* The file with the stations is in an FTP server and we want to store it
+ locally in separate folders for each day that the code is run.
+* The problem is that the ``stations.zip`` file is updated on the server
+ instead of creating a new one, so the hash check would fail.
+
+This is how you can solve this problem:
+
+.. code:: python
+
+ import datetime
+ import pooch
+
+ # Get the current data to store the files in separate folders
+ CURRENT_DATE = datetime.datetime.now().date()
+
+ GOODBOY = pooch.create(
+ path=pooch.os_cache("bom_daily_stations") / CURRENT_DATE,
+ base_url="ftp://ftp.bom.gov.au/anon2/home/ncc/metadata/sitelists/",
+ registry={
+ "stations.zip": None,
+ },
+ )
+
+When running this same code again at a different date, the file will be
+downloaded again because the local cache folder changed and the file is no
+longer present in it.
+If you omit ``CURRENT_DATE`` from the cache path, then Pooch will only fetch
+the files once, unless they are deleted from the cache.
+
+.. attention::
+
+ If this script is run over a period of time, your cache directory will
+ increase in size, as the files are stored in daily subdirectories.
+
+
+.. _hashes-other:
+
+Other supported hashes
+----------------------
+
+Beyond hashing algorithms supported by ``hashlib``, Pooch supports algorithms
+provided by the `xxhash package `__.
+If the ``xxhash`` package is available, users may specify to use one of
+the algorithms provided by the package.
+
+.. code:: bash
+
+ $ xxh128sum data/store.zip
+ 6a71973c93eac6c8839ce751ce10ae48 data/store.zip
+ $ # ^^^^^^^^^^^^^^^^^^^ The hash ^^^^^^^^^^^^^^ The filename
+
+.. code:: python
+
+ import datetime
+ import pooch
+
+ # Get the current data to store the files in separate folders
+ CURRENT_DATE = datetime.datetime.now().date()
+
+ GOODBOY = pooch.create(
+ [...],
+ registry={
+ "store.zip": "xxh128:6a71973c93eac6c8839ce751ce10ae48",
+ },
+ )
diff --git a/v1.8.1/_sources/index.rst b/v1.8.1/_sources/index.rst
new file mode 100644
index 00000000..f2306777
--- /dev/null
+++ b/v1.8.1/_sources/index.rst
@@ -0,0 +1,158 @@
+.. title:: Home
+
+.. raw:: html
+
+
+
+
Pooch
+
+
+ A friend to fetch your data files
+
+
+
+
+
+.. raw:: html
+
+
+ Just want to download a file without messing with
+ requests and urllib?
+ Trying to add sample datasets to your Python package?
+ Pooch is here to help!
+
+