From 823d3e43e1fe507c00daa7455ad5566669480efc Mon Sep 17 00:00:00 2001 From: philnagel Date: Wed, 22 Oct 2025 10:55:53 -0500 Subject: [PATCH 1/7] feat(progress): add progress bar feature via tqdm --- contextily/__init__.py | 1 + contextily/progress.py | 33 ++++++++++++++++ contextily/tile.py | 9 +++-- notebooks/intro_guide.ipynb | 75 ++++++++++++++++++++++--------------- 4 files changed, 85 insertions(+), 33 deletions(-) create mode 100644 contextily/progress.py diff --git a/contextily/__init__.py b/contextily/__init__.py index a64a2130..a15bb1c8 100644 --- a/contextily/__init__.py +++ b/contextily/__init__.py @@ -6,6 +6,7 @@ from .place import Place, plot_map from .tile import * from .plotting import add_basemap, add_attribution +from .progress import set_progress_bar from importlib.metadata import PackageNotFoundError, version diff --git a/contextily/progress.py b/contextily/progress.py new file mode 100644 index 00000000..28291148 --- /dev/null +++ b/contextily/progress.py @@ -0,0 +1,33 @@ +from tqdm import tqdm as _default_progress_bar +from contextlib import nullcontext + +# Default progress bar class (can be changed by set_progress_bar) +_progress_bar = _default_progress_bar + +def set_progress_bar(progress_bar=None): + """ + Set the progress bar class to be used for downloading tiles. + + Parameters + ---------- + progress_bar : class, optional + A tqdm-compatible progress bar class. If None, uses the default tqdm. + The progress bar class should have the same interface as tqdm. + Common alternatives include: + - tqdm.notebook.tqdm for Jupyter notebooks + - custom implementations with the same interface + """ + global _progress_bar + _progress_bar = progress_bar if progress_bar is not None else _default_progress_bar + + +def get_progress_bar(): + """ + Returns the progress bar class to be used for downloading tiles. + + Returns + ---------- + progress_bar : class + A tqdm-compatible progress bar class. + """ + return _progress_bar diff --git a/contextily/tile.py b/contextily/tile.py index 2e64ac31..1ea5ae5b 100644 --- a/contextily/tile.py +++ b/contextily/tile.py @@ -19,6 +19,7 @@ from joblib import Memory as _Memory from joblib import Parallel, delayed from rasterio.transform import from_origin +from .progress import get_progress_bar from rasterio.io import MemoryFile from rasterio.vrt import WarpedVRT from rasterio.enums import Resampling @@ -280,9 +281,11 @@ def bounds2img( "threads" if (n_connections == 1 or not use_cache) else "processes" ) fetch_tile_fn = memory.cache(_fetch_tile) if use_cache else _fetch_tile - arrays = Parallel(n_jobs=n_connections, prefer=preferred_backend)( - delayed(fetch_tile_fn)(tile_url, wait, max_retries) for tile_url in tile_urls - ) + + with tqdm_joblib(get_progress_bar()(total=len(tile_urls), desc="Downloading tiles")) as progress_bar: + arrays = Parallel(n_jobs=n_connections, prefer=preferred_backend)( + delayed(fetch_tile_fn)(tile_url, wait, max_retries) for tile_url in tile_urls + ) # merge downloaded tiles merged, extent = _merge_tiles(tiles, arrays) # lon/lat extent --> Spheric Mercator diff --git a/notebooks/intro_guide.ipynb b/notebooks/intro_guide.ipynb index 90f7bf74..c0790267 100644 --- a/notebooks/intro_guide.ipynb +++ b/notebooks/intro_guide.ipynb @@ -11,7 +11,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -34,7 +34,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -50,7 +50,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -68,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -103,7 +103,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -138,7 +138,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -18614,7 +18614,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -18985,7 +18985,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -19006,7 +19006,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -19028,7 +19028,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19055,7 +19055,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19082,7 +19082,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19122,7 +19122,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -19139,7 +19139,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19180,7 +19180,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19198,7 +19198,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19216,7 +19216,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19253,7 +19253,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -19269,7 +19269,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19297,7 +19297,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -19321,7 +19321,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19353,7 +19353,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19390,7 +19390,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19435,7 +19435,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19462,7 +19462,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19489,7 +19489,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19516,7 +19516,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19553,7 +19553,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19582,7 +19582,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19612,7 +19612,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -19631,6 +19631,21 @@ "cx.add_basemap(ax, crs=df.crs.to_string(), source=cx.providers.CartoDB.Positron, zoom=12)\n", "cx.add_basemap(ax, crs=df.crs.to_string(), source=cx.providers.CartoDB.PositronOnlyLabels, zoom=10)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Progress bars\n", + "\n", + "The tile download progress bar uses [`tqdm`](https://tqdm.github.io/) to display a progress bar. You can override the default `tqdm` instance. For example if you are running in a Jupyter Notebook environment, you might want to use `tqdm.notebook.tqdm` instead of the default `tqdm.tqdm`. You can also build custom `tqdm` implementations.\n", + "\n", + "```python\n", + "import contextily as cx\n", + "from tqdm.notebook import tqdm\n", + "cx.set_progress_bar(tqdm)\n", + "```" + ] } ], "metadata": { From 481dd6bf1cca79d4fffe0a0253ff44ef09bd366a Mon Sep 17 00:00:00 2001 From: philnagel Date: Wed, 22 Oct 2025 11:01:56 -0500 Subject: [PATCH 2/7] fix(progress): update accidentally committed experiment --- contextily/tile.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/contextily/tile.py b/contextily/tile.py index 1ea5ae5b..4f441984 100644 --- a/contextily/tile.py +++ b/contextily/tile.py @@ -282,10 +282,16 @@ def bounds2img( ) fetch_tile_fn = memory.cache(_fetch_tile) if use_cache else _fetch_tile - with tqdm_joblib(get_progress_bar()(total=len(tile_urls), desc="Downloading tiles")) as progress_bar: + with get_progress_bar()(total=len(tile_urls), desc="Downloading tiles") as pbar: + def fetch_with_progress(tile_url): + result = fetch_tile_fn(tile_url, wait, max_retries) + pbar.update(1) + return result + arrays = Parallel(n_jobs=n_connections, prefer=preferred_backend)( - delayed(fetch_tile_fn)(tile_url, wait, max_retries) for tile_url in tile_urls + delayed(fetch_with_progress)(tile_url) for tile_url in tile_urls ) + # merge downloaded tiles merged, extent = _merge_tiles(tiles, arrays) # lon/lat extent --> Spheric Mercator From ba69c0d22e9de10596571e8a4b0409c29ae20abb Mon Sep 17 00:00:00 2001 From: philnagel Date: Wed, 22 Oct 2025 11:06:07 -0500 Subject: [PATCH 3/7] build(progress): add tqdm dependency --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 913367ad..7fa8ba7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,8 @@ dependencies = [ "rasterio", "requests", "joblib", - "xyzservices" + "xyzservices", + "tqdm" ] [project.urls] From 68e91a8b32d1715b391d7fdb526d03288593aace Mon Sep 17 00:00:00 2001 From: philnagel Date: Thu, 23 Oct 2025 09:38:29 -0500 Subject: [PATCH 4/7] ci: add new tqdm dependency in ci files --- ci/envs/310-conda-forge.yaml | 1 + ci/envs/311-conda-forge.yaml | 1 + ci/envs/312-latest-conda-forge.yaml | 1 + ci/envs/313-latest-conda-forge.yaml | 1 + 4 files changed, 4 insertions(+) diff --git a/ci/envs/310-conda-forge.yaml b/ci/envs/310-conda-forge.yaml index 8929971b..ab40f2cf 100644 --- a/ci/envs/310-conda-forge.yaml +++ b/ci/envs/310-conda-forge.yaml @@ -12,6 +12,7 @@ dependencies: - requests - joblib - xyzservices + - tqdm # testing - pip - pytest diff --git a/ci/envs/311-conda-forge.yaml b/ci/envs/311-conda-forge.yaml index 8b9c6983..7cb429b2 100644 --- a/ci/envs/311-conda-forge.yaml +++ b/ci/envs/311-conda-forge.yaml @@ -12,6 +12,7 @@ dependencies: - requests - joblib - xyzservices + - tqdm # testing - pip - pytest diff --git a/ci/envs/312-latest-conda-forge.yaml b/ci/envs/312-latest-conda-forge.yaml index 4fbffb4e..021a88d8 100644 --- a/ci/envs/312-latest-conda-forge.yaml +++ b/ci/envs/312-latest-conda-forge.yaml @@ -12,6 +12,7 @@ dependencies: - requests - joblib - xyzservices + - tqdm # testing - pip - pytest diff --git a/ci/envs/313-latest-conda-forge.yaml b/ci/envs/313-latest-conda-forge.yaml index 0989bde7..b3e62007 100644 --- a/ci/envs/313-latest-conda-forge.yaml +++ b/ci/envs/313-latest-conda-forge.yaml @@ -12,6 +12,7 @@ dependencies: - requests - joblib - xyzservices + - tqdm # testing - pip - pytest From cf06b92d5d39852ba3b8aa9fced68f0cd1bc844c Mon Sep 17 00:00:00 2001 From: philnagel Date: Thu, 23 Oct 2025 09:40:49 -0500 Subject: [PATCH 5/7] feat: use ThreadPoolExecutor instead of joblib there are known issues with using tqdm along with some joblib backends. using ThreadPoolExecutor instead. --- contextily/progress.py | 31 +++++++++++++++++++++++++++---- contextily/tile.py | 36 +++++++++++++++++++++--------------- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/contextily/progress.py b/contextily/progress.py index 28291148..2fd5c993 100644 --- a/contextily/progress.py +++ b/contextily/progress.py @@ -11,23 +11,46 @@ def set_progress_bar(progress_bar=None): Parameters ---------- progress_bar : class, optional - A tqdm-compatible progress bar class. If None, uses the default tqdm. + A tqdm-compatible progress bar class. If None, progress bar is disabled. The progress bar class should have the same interface as tqdm. Common alternatives include: - tqdm.notebook.tqdm for Jupyter notebooks - custom implementations with the same interface """ global _progress_bar - _progress_bar = progress_bar if progress_bar is not None else _default_progress_bar + _progress_bar = progress_bar def get_progress_bar(): """ Returns the progress bar class to be used for downloading tiles. + If progress bars are disabled (set to None), returns a no-op context + manager that doesn't display progress. Returns ---------- - progress_bar : class - A tqdm-compatible progress bar class. + progress_bar : callable + A callable that returns either a tqdm-compatible progress bar or a + no-op context manager with update/close methods if progress bars + are disabled. """ + if _progress_bar is None: + class NoOpProgress: + def __init__(self, *args, **kwargs): + self.context = nullcontext() + + def __enter__(self): + self.context.__enter__() + return self + + def __exit__(self, *args): + self.context.__exit__(*args) + + def update(self, n=1): + pass + + def close(self): + pass + + return lambda *args, **kwargs: NoOpProgress(*args, **kwargs) return _progress_bar diff --git a/contextily/tile.py b/contextily/tile.py index 4f441984..d53cff30 100644 --- a/contextily/tile.py +++ b/contextily/tile.py @@ -17,7 +17,6 @@ import rasterio as rio from PIL import Image, UnidentifiedImageError from joblib import Memory as _Memory -from joblib import Parallel, delayed from rasterio.transform import from_origin from .progress import get_progress_bar from rasterio.io import MemoryFile @@ -274,23 +273,30 @@ def bounds2img( # download tiles if n_connections < 1 or not isinstance(n_connections, int): raise ValueError(f"n_connections must be a positive integer value.") - # Use threads for a single connection to avoid the overhead of spawning a process. Use processes for multiple - # connections if caching is enabled, as threads lead to memory issues when used in combination with the joblib - # memory caching (used for the _fetch_tile() function). - preferred_backend = ( - "threads" if (n_connections == 1 or not use_cache) else "processes" - ) - fetch_tile_fn = memory.cache(_fetch_tile) if use_cache else _fetch_tile + fetch_tile_fn = memory.cache(_fetch_tile) if use_cache else _fetch_tile + + # Always use threads for notebook compatibility + from concurrent.futures import ThreadPoolExecutor, as_completed + + arrays = [None] * len(tile_urls) # Pre-allocate result list with get_progress_bar()(total=len(tile_urls), desc="Downloading tiles") as pbar: - def fetch_with_progress(tile_url): - result = fetch_tile_fn(tile_url, wait, max_retries) - pbar.update(1) - return result + with ThreadPoolExecutor(max_workers=n_connections) as executor: + # Submit all tasks and store futures with their indices + future_to_index = { + executor.submit(fetch_tile_fn, url, wait, max_retries): idx + for idx, url in enumerate(tile_urls) + } - arrays = Parallel(n_jobs=n_connections, prefer=preferred_backend)( - delayed(fetch_with_progress)(tile_url) for tile_url in tile_urls - ) + # Process completed futures as they finish + for future in as_completed(future_to_index): + idx = future_to_index[future] + try: + arrays[idx] = future.result() + except Exception as e: + # Re-raise any exceptions from the worker + raise e from None + pbar.update(1) # merge downloaded tiles merged, extent = _merge_tiles(tiles, arrays) From c875a100594fbb20adf0bc252e2a3940613f2b84 Mon Sep 17 00:00:00 2001 From: philnagel Date: Thu, 23 Oct 2025 09:41:43 -0500 Subject: [PATCH 6/7] style: fix import location --- contextily/tile.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/contextily/tile.py b/contextily/tile.py index d53cff30..fb7f4ae0 100644 --- a/contextily/tile.py +++ b/contextily/tile.py @@ -17,6 +17,7 @@ import rasterio as rio from PIL import Image, UnidentifiedImageError from joblib import Memory as _Memory +from concurrent.futures import ThreadPoolExecutor, as_completed from rasterio.transform import from_origin from .progress import get_progress_bar from rasterio.io import MemoryFile @@ -275,10 +276,7 @@ def bounds2img( raise ValueError(f"n_connections must be a positive integer value.") fetch_tile_fn = memory.cache(_fetch_tile) if use_cache else _fetch_tile - - # Always use threads for notebook compatibility - from concurrent.futures import ThreadPoolExecutor, as_completed - + arrays = [None] * len(tile_urls) # Pre-allocate result list with get_progress_bar()(total=len(tile_urls), desc="Downloading tiles") as pbar: with ThreadPoolExecutor(max_workers=n_connections) as executor: From 64622aab8d19d5d5aa0ae373fc45494df085ce14 Mon Sep 17 00:00:00 2001 From: philnagel Date: Thu, 23 Oct 2025 09:43:21 -0500 Subject: [PATCH 7/7] docs: update docs for disabling progress bar --- notebooks/intro_guide.ipynb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/notebooks/intro_guide.ipynb b/notebooks/intro_guide.ipynb index c0790267..5e334e7c 100644 --- a/notebooks/intro_guide.ipynb +++ b/notebooks/intro_guide.ipynb @@ -19644,7 +19644,9 @@ "import contextily as cx\n", "from tqdm.notebook import tqdm\n", "cx.set_progress_bar(tqdm)\n", - "```" + "```\n", + "\n", + "If you don't want to display any progress bar, call `cx.set_progress_bar(None)`" ] } ],