Merge pull request #88 from hynky1999/remove_release_tests

remove tests fix downloader

.github/workflows/release.yml

@@ -11,24 +11,6 @@ env:
 
 
 jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Install dependencies
-        run: pip install -r requirements.test.txt
-
-      - name: Run tests
-        run: python -m unittest discover -s tests -p "*_tests.py"  # Replace with your test command
-
-
   publish:
     needs: test
     runs-on: ubuntu-latest

cmoncrawl/processor/pipeline/downloader.py

@@ -60,10 +60,9 @@ async def wrapper(*args, **kwargs):
 
 
 class DownloadError(Exception):
-    def __init__(self, reason: str, status: int, **args: str):
+    def __init__(self, reason: str, status: int):
         self.reason = reason
         self.status = status
-        self.args = args
 
 
 class IDownloader:
@@ -121,7 +120,7 @@ async def _download_warc(
         async with self.client.get(url, headers=headers) as response:
             if not response.ok:
                 reason: str = response.reason if response.reason else "Unknown"
-                raise DownloadError(reason, response.status, **response.headers)
+                raise DownloadError(reason, response.status)
             else:
                 # will be unziped, we cannot use the stream since warcio doesn't support async
                 response_bytes = await response.content.read()
@@ -156,7 +155,7 @@ def should_retry(retry: int, reason: str, status: int, **args: str):
             try:
                 return await self._download_warc(url, headers, domain_record)
             except DownloadError as e:
-                if not should_retry(retry, f"{str(e)} {type(e)}", e.status, **e.args):
+                if not should_retry(retry, f"{str(e)} {type(e)}", e.status):
                     raise e
 
             except (