From 2fc91ef79d84336c3e8b317afa8caae86503341b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=80=D1=82=D1=8B=D0=BD=D0=BE=D0=B2=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D1=81=D0=B8=D0=BC=20=D0=A1=D0=B5=D1=80=D0=B3=D0=B5?= =?UTF-8?q?=D0=B5=D0=B2=D0=B8=D1=87?= Date: Thu, 17 Aug 2023 12:54:17 +0000 Subject: [PATCH 1/2] [DOP-8157] Bump version --- onetl/VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onetl/VERSION b/onetl/VERSION index ac39a106c..f374f6662 100644 --- a/onetl/VERSION +++ b/onetl/VERSION @@ -1 +1 @@ -0.9.0 +0.9.1 From 801fff243aa143ce310c1aa8e99ab36f306672bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=80=D1=82=D1=8B=D0=BD=D0=BE=D0=B2=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D1=81=D0=B8=D0=BC=20=D0=A1=D0=B5=D1=80=D0=B3=D0=B5?= =?UTF-8?q?=D0=B5=D0=B2=D0=B8=D1=87?= Date: Thu, 17 Aug 2023 18:44:05 +0000 Subject: [PATCH 2/2] [DOP-8157] Fix calculating number of workers in FileDownloader/Uploader/Mover --- docs/changelog/0.9.1.rst | 9 +++++++++ docs/changelog/index.rst | 1 + onetl/file/file_downloader/file_downloader.py | 2 +- onetl/file/file_mover/file_mover.py | 2 +- onetl/file/file_uploader/file_uploader.py | 2 +- 5 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 docs/changelog/0.9.1.rst diff --git a/docs/changelog/0.9.1.rst b/docs/changelog/0.9.1.rst new file mode 100644 index 000000000..c45d620f4 --- /dev/null +++ b/docs/changelog/0.9.1.rst @@ -0,0 +1,9 @@ +0.9.1 (2023-08-17) +================== + +Bug Fixes +--------- + +- Fixed bug then number of threads created by ``FileDownloader`` / ``FileUploader`` / ``FileMover`` was + not ``min(workers, len(files))``, but ``max(workers, len(files))``. leading to create too much workers + on large files list. diff --git a/docs/changelog/index.rst b/docs/changelog/index.rst index ef5b51282..4af96fada 100644 --- a/docs/changelog/index.rst +++ b/docs/changelog/index.rst @@ -4,6 +4,7 @@ DRAFT NEXT_RELEASE + 0.9.1 0.9.0 0.8.1 0.8.0 diff --git a/onetl/file/file_downloader/file_downloader.py b/onetl/file/file_downloader/file_downloader.py index 7bcbea58b..ba7c9b18a 100644 --- a/onetl/file/file_downloader/file_downloader.py +++ b/onetl/file/file_downloader/file_downloader.py @@ -719,7 +719,7 @@ def _bulk_download( if workers > 1: with ThreadPoolExecutor( - max_workers=max(workers, len(to_download)), + max_workers=min(workers, len(to_download)), thread_name_prefix=self.__class__.__name__, ) as executor: futures = [ diff --git a/onetl/file/file_mover/file_mover.py b/onetl/file/file_mover/file_mover.py index 51b80fe70..9c28632a3 100644 --- a/onetl/file/file_mover/file_mover.py +++ b/onetl/file/file_mover/file_mover.py @@ -516,7 +516,7 @@ def _bulk_move( if workers > 1: with ThreadPoolExecutor( - max_workers=max(workers, len(to_move)), + max_workers=min(workers, len(to_move)), thread_name_prefix=self.__class__.__name__, ) as executor: futures = [ diff --git a/onetl/file/file_uploader/file_uploader.py b/onetl/file/file_uploader/file_uploader.py index 30b4e6ca2..811c27c5f 100644 --- a/onetl/file/file_uploader/file_uploader.py +++ b/onetl/file/file_uploader/file_uploader.py @@ -544,7 +544,7 @@ def _bulk_upload( if workers > 1: with ThreadPoolExecutor( - max_workers=max(workers, len(to_upload)), + max_workers=min(workers, len(to_upload)), thread_name_prefix=self.__class__.__name__, ) as executor: futures = [