fix docip bug & add support for python 3.11 (#205)

Python3WebSpider · Dec 26, 2023 · 97229e6 · 97229e6
1 parent 2344ee1
commit 97229e6
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/proxypool/crawlers/public/docip.py b/proxypool/crawlers/public/docip.py
@@ -25,7 +25,7 @@ def parse(self, html):
             proxy_list = result['data']
             for proxy_item in proxy_list:
                 host = proxy_item['ip']
-                port = proxy_item['port']
+                port = host.split(':')[-1]
                 yield Proxy(host=host, port=port)
         except json.JSONDecodeError:
             print("json.JSONDecodeError")

diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py
@@ -82,7 +82,7 @@ def run(self):
             logger.debug(f'testing proxies use cursor {cursor}, count {TEST_BATCH}')
             cursor, proxies = self.redis.batch(cursor, count=TEST_BATCH)
             if proxies:
-                tasks = [self.test(proxy) for proxy in proxies]
+                tasks = [self.loop.create_task(self.test(proxy)) for proxy in proxies]
                 self.loop.run_until_complete(asyncio.wait(tasks))
             if not cursor:
                 break

diff --git a/requirements.txt b/requirements.txt
@@ -11,7 +11,7 @@ redis>=3.5.3,<4.0.0
 lxml>=4.6.5,<5.0.0
 fake_headers>=1.0.2,<2.0.0
 maxminddb_geolite2==2018.703
-gevent>=21.8.0,<22.0.0
+gevent>=21.8.0,<24.0.0
 tornado>=6.0,<7.0
 itsdangerous==0.24
 MarkupSafe<2.1.0