[DOP-8157] Fix calculating number of workers in FileDownloader/Upload…

…er/Mover
MobileTeleSystems · Aug 17, 2023 · 801fff2 · 801fff2
1 parent 2fc91ef
commit 801fff2
Show file tree

Hide file tree

Showing 5 changed files with 13 additions and 3 deletions.
diff --git a/docs/changelog/0.9.1.rst b/docs/changelog/0.9.1.rst
@@ -0,0 +1,9 @@
+0.9.1 (2023-08-17)
+==================
+
+Bug Fixes
+---------
+
+- Fixed bug then number of threads created by ``FileDownloader`` / ``FileUploader`` / ``FileMover`` was
+  not ``min(workers, len(files))``, but ``max(workers, len(files))``. leading to create too much workers
+  on large files list.
diff --git a/docs/changelog/index.rst b/docs/changelog/index.rst
@@ -4,6 +4,7 @@
 
     DRAFT
     NEXT_RELEASE
+    0.9.1
     0.9.0
     0.8.1
     0.8.0

diff --git a/onetl/file/file_downloader/file_downloader.py b/onetl/file/file_downloader/file_downloader.py
@@ -719,7 +719,7 @@ def _bulk_download(
 
         if workers > 1:
             with ThreadPoolExecutor(
-                max_workers=max(workers, len(to_download)),
+                max_workers=min(workers, len(to_download)),
                 thread_name_prefix=self.__class__.__name__,
             ) as executor:
                 futures = [

diff --git a/onetl/file/file_mover/file_mover.py b/onetl/file/file_mover/file_mover.py
@@ -516,7 +516,7 @@ def _bulk_move(
 
         if workers > 1:
             with ThreadPoolExecutor(
-                max_workers=max(workers, len(to_move)),
+                max_workers=min(workers, len(to_move)),
                 thread_name_prefix=self.__class__.__name__,
             ) as executor:
                 futures = [

diff --git a/onetl/file/file_uploader/file_uploader.py b/onetl/file/file_uploader/file_uploader.py
@@ -544,7 +544,7 @@ def _bulk_upload(
 
         if workers > 1:
             with ThreadPoolExecutor(
-                max_workers=max(workers, len(to_upload)),
+                max_workers=min(workers, len(to_upload)),
                 thread_name_prefix=self.__class__.__name__,
             ) as executor:
                 futures = [
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,6 +4,7 @@ @@
         DRAFT
         NEXT_RELEASE
+.9.1
 .9.0
 .8.1
 .8.0
@@ Expand Down @@