Avoid loading entire files into memory when downloading from ESGF (#2434

)
ESMValGroup · May 24, 2024 · 53bb9ac · 53bb9ac
1 parent 88e53ac
commit 53bb9ac
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/esmvalcore/esgf/_download.py b/esmvalcore/esgf/_download.py
@@ -466,7 +466,10 @@ def _download(self, local_file, url):
                                 cert=get_credentials())
         response.raise_for_status()
         with tmp_file.open("wb") as file:
-            for chunk in response.iter_content(chunk_size=None):
+            # Specify chunk_size to avoid
+            # https://github.com/psf/requests/issues/5536
+            megabyte = 2**20
+            for chunk in response.iter_content(chunk_size=megabyte):
                 if hasher is not None:
                     hasher.update(chunk)
                 file.write(chunk)

diff --git a/tests/unit/esgf/test_download.py b/tests/unit/esgf/test_download.py
@@ -459,7 +459,7 @@ def test_single_download(mocker, tmp_path, checksum):
     # We checked for a valid response
     response.raise_for_status.assert_called_once()
     # And requested a reasonable chunk size
-    response.iter_content.assert_called_with(chunk_size=None)
+    response.iter_content.assert_called_with(chunk_size=2**20)
 
 
 def test_download_skip_existing(tmp_path, caplog):