add a --hard-retries option (#30)

* feat: add a --hard-retries option * fix: don't cut off the help message for --retries in the middle of a sentence * move hard_reties config to OtherConfig * ci: fix --------- Co-authored-by: yzqzss <[email protected]>
saveweb · Oct 9, 2024 · ffed938 · ffed938
1 parent db2818c
commit ffed938
Show file tree

Hide file tree

Showing 5 changed files with 14 additions and 6 deletions.
diff --git a/.github/workflows/test-dumogenerator.yml b/.github/workflows/test-dumogenerator.yml
@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.12", "3.13.0-rc.1"]
+        python-version: ["3.8", "3.12", "3.13"]
 
     steps:
     - uses: actions/checkout@v4
@@ -40,4 +40,4 @@ jobs:
         python -m wikiteam3.dumpgenerator -h
     - name: Test with pytest
       run: |
-        cd wikiteam3/dumpgenerator && pytest && cd ../../
+        pytest
diff --git a/wikiteam3/dumpgenerator/api/page_titles.py b/wikiteam3/dumpgenerator/api/page_titles.py
@@ -27,7 +27,7 @@ def getPageTitlesAPI(config: Config, session: requests.Session):
     delay_session = SessionMonkeyPatch(
             session=session, config=config,
             add_delay=True, delay_msg="Session delay: "+__name__,
-            hard_retries=3
+            hard_retries=3 # TODO: --hard-retries
         )
     delay_session.hijack()
     for namespace in namespaces:

diff --git a/wikiteam3/dumpgenerator/cli/cli.py b/wikiteam3/dumpgenerator/cli/cli.py
@@ -49,7 +49,10 @@ def getArgumentParser():
         "if you wanna reuse the connection]"
     )
     parser.add_argument(
-        "--retries", metavar="5", default=5, help="Maximum number of retries for "
+        "--retries", metavar="5", default=5, help="Maximum number of retries for each request before failing."
+    )
+    parser.add_argument(
+        "--hard-retries", metavar="3", default=3, help="Maximum number of hard retries for each request before failing. (for now, this only controls the hard retries during images downloading)"
     )
     parser.add_argument("--path", help="path to store wiki dump at")
     parser.add_argument(
@@ -290,7 +293,7 @@ def get_parameters(params=None) -> Tuple[Config, OtherConfig]:
     # Create session
     mod_requests_text(requests) # monkey patch # type: ignore
     session = requests.Session()
-    patch_sess = SessionMonkeyPatch(session=session, hard_retries=1)
+    patch_sess = SessionMonkeyPatch(session=session, hard_retries=1) # hard retry once to avoid spending too much time on initial detection
     patch_sess.hijack()
     def print_request(r: requests.Response, *args, **kwargs):
         # TODO: use logging
@@ -548,6 +551,8 @@ def sleep(self, response=None):
         assert_max_images = args.assert_max_images,
         assert_max_images_bytes = args.assert_max_images_bytes,
 
+        hard_retries = int(args.hard_retries),
+
         upload = args.upload,
         uploader_args = args.uploader_args,
     )

diff --git a/wikiteam3/dumpgenerator/config.py b/wikiteam3/dumpgenerator/config.py
@@ -113,5 +113,8 @@ class OtherConfig:
     assert_max_images: Optional[int] 
     assert_max_images_bytes: Optional[int] 
 
+    hard_retries: int
+    """ Number of hard retries """
+
     upload: bool 
     uploader_args: List[str]
diff --git a/wikiteam3/dumpgenerator/dump/image/image.py b/wikiteam3/dumpgenerator/dump/image/image.py
@@ -107,7 +107,7 @@ def modify_headers(headers: Optional[Dict] = None) -> Dict:
             return headers
 
 
-        patch_sess = SessionMonkeyPatch(session=session, config=config, hard_retries=3)
+        patch_sess = SessionMonkeyPatch(session=session, config=config, hard_retries=other.hard_retries)
         patch_sess.hijack()
 
         ia_session = requests.Session()