Merge pull request #100 from spacewaterbear/main

max_requests_per_second cannot be zero otherwise it will break here :…
hynky1999 · Feb 13, 2024 · b324fe2 · b324fe2
2 parents f8034f3 + 5b3ddd4
commit b324fe2
Showing 1 changed file with 1 addition and 4 deletions.
diff --git a/cmoncrawl/integrations/download.py b/cmoncrawl/integrations/download.py
@@ -338,9 +338,6 @@ def run_download(args: argparse.Namespace):
     max_crawls_per_file = (
         args.max_crawls_per_file if mode == DownloadOutputFormat.RECORD else 1
     )
-    max_requests_per_second = (
-        args.max_requests_per_second if mode == DownloadOutputFormat.RECORD else 0
-    )
     # HTML exlusives
     encoding = args.encoding if mode == DownloadOutputFormat.HTML else None
     download_method = (
@@ -357,7 +354,7 @@ def run_download(args: argparse.Namespace):
             limit=args.limit,
             max_retry=args.max_retry,
             sleep_base=args.sleep_base,
-            max_requests_per_second=max_requests_per_second,
+            max_requests_per_second=args.max_requests_per_second,
             mode=mode,
             max_crawls_per_file=max_crawls_per_file,
             encoding=encoding,