diff --git a/proxypool/crawlers/public/docip.py b/proxypool/crawlers/public/docip.py index 070c598..154871f 100644 --- a/proxypool/crawlers/public/docip.py +++ b/proxypool/crawlers/public/docip.py @@ -25,7 +25,7 @@ def parse(self, html): proxy_list = result['data'] for proxy_item in proxy_list: host = proxy_item['ip'] - port = proxy_item['port'] + port = host.split(':')[-1] yield Proxy(host=host, port=port) except json.JSONDecodeError: print("json.JSONDecodeError") diff --git a/proxypool/processors/tester.py b/proxypool/processors/tester.py index 353332a..5879528 100644 --- a/proxypool/processors/tester.py +++ b/proxypool/processors/tester.py @@ -82,7 +82,7 @@ def run(self): logger.debug(f'testing proxies use cursor {cursor}, count {TEST_BATCH}') cursor, proxies = self.redis.batch(cursor, count=TEST_BATCH) if proxies: - tasks = [self.test(proxy) for proxy in proxies] + tasks = [self.loop.create_task(self.test(proxy)) for proxy in proxies] self.loop.run_until_complete(asyncio.wait(tasks)) if not cursor: break diff --git a/requirements.txt b/requirements.txt index c9407c7..33f35c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ redis>=3.5.3,<4.0.0 lxml>=4.6.5,<5.0.0 fake_headers>=1.0.2,<2.0.0 maxminddb_geolite2==2018.703 -gevent>=21.8.0,<22.0.0 +gevent>=21.8.0,<24.0.0 tornado>=6.0,<7.0 itsdangerous==0.24 MarkupSafe<2.1.0