From 258b6647c8227b0761daca82b5c1be1bf72cf737 Mon Sep 17 00:00:00 2001
From: IryNeko <31180530+IryNeko@users.noreply.github.com>
Date: Sun, 6 Oct 2024 11:46:04 +0800
Subject: [PATCH 1/6] make it possible to skip video file extension
---
waifuc/source/web.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/waifuc/source/web.py b/waifuc/source/web.py
index 76ccbc3a..67a1d6f5 100644
--- a/waifuc/source/web.py
+++ b/waifuc/source/web.py
@@ -27,6 +27,7 @@ def __init__(self, group_name: str, session: httpx.Client = None, download_silen
self.download_silent = download_silent
self.session = session or get_requests_session()
self.group_name = group_name
+ self.skip_video = False
@classmethod
def _rate_limiter(cls) -> Limiter:
@@ -57,7 +58,9 @@ def _iter(self) -> Iterator[ImageItem]:
_, ext_name = os.path.splitext(urlsplit(url).filename)
filename = f'{self.group_name}_{id_}{ext_name}'
td_file = os.path.join(td, filename)
-
+ if self.skip_video and any(x in filename for x.lower() in [".webm",".mp4",".gif",".mov",".avi",".avchd",".flv"]):
+ warnings.warn(f'Skipped due to skip_video setting')
+ continue
try:
self._rate_limiter().try_acquire(filename)
download_file(
From 19ee44c3a8f6a51e99a5ad42e8edc362ffbb1ac4 Mon Sep 17 00:00:00 2001
From: IryNeko <31180530+IryNeko@users.noreply.github.com>
Date: Sun, 6 Oct 2024 11:50:02 +0800
Subject: [PATCH 2/6] Update web.py
---
waifuc/source/web.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/waifuc/source/web.py b/waifuc/source/web.py
index 67a1d6f5..3ed92af3 100644
--- a/waifuc/source/web.py
+++ b/waifuc/source/web.py
@@ -58,7 +58,7 @@ def _iter(self) -> Iterator[ImageItem]:
_, ext_name = os.path.splitext(urlsplit(url).filename)
filename = f'{self.group_name}_{id_}{ext_name}'
td_file = os.path.join(td, filename)
- if self.skip_video and any(x in filename for x.lower() in [".webm",".mp4",".gif",".mov",".avi",".avchd",".flv"]):
+ if self.skip_video and any(x.lower() in filename for x in [".webm",".mp4",".gif",".mov",".avi",".avchd",".flv"]):
warnings.warn(f'Skipped due to skip_video setting')
continue
try:
From baf0633e56774a2cec0d420e506d83706e4d1a9b Mon Sep 17 00:00:00 2001
From: IryNeko <31180530+IryNeko@users.noreply.github.com>
Date: Sun, 6 Oct 2024 11:52:27 +0800
Subject: [PATCH 3/6] Update web.py
---
waifuc/source/web.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/waifuc/source/web.py b/waifuc/source/web.py
index 3ed92af3..1b2962e0 100644
--- a/waifuc/source/web.py
+++ b/waifuc/source/web.py
@@ -58,7 +58,7 @@ def _iter(self) -> Iterator[ImageItem]:
_, ext_name = os.path.splitext(urlsplit(url).filename)
filename = f'{self.group_name}_{id_}{ext_name}'
td_file = os.path.join(td, filename)
- if self.skip_video and any(x.lower() in filename for x in [".webm",".mp4",".gif",".mov",".avi",".avchd",".flv"]):
+ if self.skip_video and any(x in filename.lower() for x in [".webm",".mp4",".gif",".mov",".avi",".avchd",".flv"]):
warnings.warn(f'Skipped due to skip_video setting')
continue
try:
From 80edeb833d0aae52310e3533c2a7ddbccbdc6045 Mon Sep 17 00:00:00 2001
From: BrianBu47 <9915476+brianbu47@user.noreply.gitee.com>
Date: Sun, 6 Oct 2024 14:44:24 +0800
Subject: [PATCH 4/6] modified: waifuc/source/web.py
---
waifuc/source/web.py | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/waifuc/source/web.py b/waifuc/source/web.py
index 1b2962e0..97164082 100644
--- a/waifuc/source/web.py
+++ b/waifuc/source/web.py
@@ -27,7 +27,7 @@ def __init__(self, group_name: str, session: httpx.Client = None, download_silen
self.download_silent = download_silent
self.session = session or get_requests_session()
self.group_name = group_name
- self.skip_video = False
+ self.file_ext_filter = None # not filter file extension as default
@classmethod
def _rate_limiter(cls) -> Limiter:
@@ -58,9 +58,13 @@ def _iter(self) -> Iterator[ImageItem]:
_, ext_name = os.path.splitext(urlsplit(url).filename)
filename = f'{self.group_name}_{id_}{ext_name}'
td_file = os.path.join(td, filename)
- if self.skip_video and any(x in filename.lower() for x in [".webm",".mp4",".gif",".mov",".avi",".avchd",".flv"]):
- warnings.warn(f'Skipped due to skip_video setting')
- continue
+ if self.file_ext_filter:
+ file_ext=filename.split(".")[-1]
+ if any(x.replace(".","").lower() in file_ext.lower() for x in self.file_ext_filter):
+ pass # only want the extension i want
+ else:
+ warnings.warn(f'Skipped due to file extension download filter')
+ continue
try:
self._rate_limiter().try_acquire(filename)
download_file(
From e13d713e56ac85a9964d79f9b066e55f6aca18eb Mon Sep 17 00:00:00 2001
From: BrianBu47 <9915476+brianbu47@user.noreply.gitee.com>
Date: Sun, 6 Oct 2024 14:46:43 +0800
Subject: [PATCH 5/6] modified: waifuc/source/web.py
---
waifuc/source/web.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/waifuc/source/web.py b/waifuc/source/web.py
index 97164082..b54fd0d8 100644
--- a/waifuc/source/web.py
+++ b/waifuc/source/web.py
@@ -58,6 +58,7 @@ def _iter(self) -> Iterator[ImageItem]:
_, ext_name = os.path.splitext(urlsplit(url).filename)
filename = f'{self.group_name}_{id_}{ext_name}'
td_file = os.path.join(td, filename)
+
if self.file_ext_filter:
file_ext=filename.split(".")[-1]
if any(x.replace(".","").lower() in file_ext.lower() for x in self.file_ext_filter):
From 2f86353109a97930f4d87c21c01efa5ecaa7577d Mon Sep 17 00:00:00 2001
From: BrianBu47 <9915476+brianbu47@user.noreply.gitee.com>
Date: Sun, 6 Oct 2024 14:47:29 +0800
Subject: [PATCH 6/6] modified: waifuc/source/web.py
---
waifuc/source/web.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/waifuc/source/web.py b/waifuc/source/web.py
index b54fd0d8..a257e1b2 100644
--- a/waifuc/source/web.py
+++ b/waifuc/source/web.py
@@ -58,7 +58,6 @@ def _iter(self) -> Iterator[ImageItem]:
_, ext_name = os.path.splitext(urlsplit(url).filename)
filename = f'{self.group_name}_{id_}{ext_name}'
td_file = os.path.join(td, filename)
-
if self.file_ext_filter:
file_ext=filename.split(".")[-1]
if any(x.replace(".","").lower() in file_ext.lower() for x in self.file_ext_filter):
@@ -66,6 +65,7 @@ def _iter(self) -> Iterator[ImageItem]:
else:
warnings.warn(f'Skipped due to file extension download filter')
continue
+
try:
self._rate_limiter().try_acquire(filename)
download_file(