Rotating proxy functionality

config.example.yml

@@ -17,6 +17,8 @@
     TSL: -4
     # Trailing Take Profit
     TTP: 2
+    # Enable rotating proxy functionality
+    ROTATING_PROXY: False
   LOGGING:
     # Logging levels used in this program are ERROR, INFO, and DEBUG
     LOG_LEVEL: INFO

requirements.txt

@@ -2,3 +2,4 @@
 requests==2.25.1
 gate_api==4.22.2
 PyYAML==6.0
+PySocks==1.7.1

src/gateio_new_coins_announcements_bot/announcement_scrapers/binance_scraper.py

@@ -4,6 +4,8 @@
 import requests
 
 from gateio_new_coins_announcements_bot.logger import logger
+from gateio_new_coins_announcements_bot.rotating_proxy import get_proxy
+from gateio_new_coins_announcements_bot.rotating_proxy import is_ready as rotating_proxy_is_ready
 from gateio_new_coins_announcements_bot.util.random import random_int
 from gateio_new_coins_announcements_bot.util.random import random_str
 
@@ -18,7 +20,17 @@ def fetch_latest_announcement(self):
         """
         logger.debug("Pulling announcement page")
         request_url = self.__request_url()
-        response = self.http_client.get(request_url)
+
+        if rotating_proxy_is_ready():
+            proxy = get_proxy()
+            logger.debug(f"Using proxy: {proxy}")
+            try:
+                response = self.http_client.get(request_url, proxies={"http": "socks5://" + proxy})
+
+            except Exception as e:
+                logger.error(e)
+        else:
+            response = self.http_client.get(request_url)
 
         # Raise an HTTPError if status is not 200
         response.raise_for_status()

src/gateio_new_coins_announcements_bot/main.py

@@ -17,6 +17,9 @@
 from gateio_new_coins_announcements_bot.trade_client import get_last_price
 from gateio_new_coins_announcements_bot.trade_client import is_api_key_valid
 from gateio_new_coins_announcements_bot.trade_client import place_order
+from gateio_new_coins_announcements_bot.rotating_proxy import init_proxy as init_rotating_proxy
+from gateio_new_coins_announcements_bot.rotating_proxy import set_proxy_event
+
 
 # To add a coin to ignore, add it to the json array in old_coins.json
 globals.old_coins = load_old_coins()
@@ -42,6 +45,10 @@
 else:
     session = {}
 
+if config["TRADE_OPTIONS"]["ROTATING_PROXY"]:
+    # Init proxy fetching
+    init_rotating_proxy()
+
 # Keep the supported currencies loaded in RAM so no time is wasted fetching
 # currencies.json from disk when an announcement is made
 logger.debug("Starting get_all_currencies")
@@ -508,6 +515,8 @@ def main():
         search_and_update()
     except KeyboardInterrupt:
         logger.info("Stopping Threads")
+        if config["TRADE_OPTIONS"]["ROTATING_PROXY"]:
+            set_proxy_event()
         globals.stop_threads = True
         globals.buy_ready.set()
         globals.sell_ready.set()

src/gateio_new_coins_announcements_bot/new_listings_scraper.py

@@ -15,6 +15,7 @@
 from gateio_new_coins_announcements_bot.logger import logger
 from gateio_new_coins_announcements_bot.store_order import load_order
 
+
 config = load_config("config.yml")
 client = load_gateio_creds("auth/auth.yml")
 spot_api = SpotApi(ApiClient(client))

src/gateio_new_coins_announcements_bot/rotating_proxy.py

@@ -0,0 +1,107 @@
+import itertools
+import socket
+import struct
+import threading
+import time
+from typing import Callable
+
+import requests
+
+import gateio_new_coins_announcements_bot.globals as globals
+from gateio_new_coins_announcements_bot.logger import logger
+
+
+_proxy_list = {}
+_proxy = None
+_event = threading.Event()
+
+
+def init_proxy():
+    threading.Thread(target=lambda: _every(60 * 10, _fetch_proxies)).start()
+    # Required for populating the proxy list when starting bot
+    _fetch_proxies()
+
+
+def _fetch_proxies():
+    _proxy_list
+    global _proxy
+    threads: list[threading.Thread] = []
+    try:
+        proxy_res = requests.get(
+            "https://www.proxyscan.io/api/proxy?last_check=180&limit=20&type=socks5&format=txt&ping=1000"
+        ).text
+    except requests.exceptions.RequestException as e:
+        logger.debug(f"Can't fetch proxies. Reason: {e}")
+        return
+
+    # Merging old proxies with new ones
+    _merged_proxies = list(proxy_res[:-1].split("\n") | _proxy_list.keys())
+
+    if len(_merged_proxies) > 0:
+        for p in _merged_proxies:
+            t = threading.Thread(target=checker, args=[p])
+            t.start()
+            threads.append(t)
+
+        for t in threads:
+            t.join()
+
+    logger.info(f"Fetched {len(_proxy_list)} proxies")
+    _proxy = itertools.cycle(_proxy_list.copy().keys())
+
+
+def get_proxy() -> str:
+    try:
+        return next(_proxy)
+    except StopIteration as exc:
+        raise Exception("No proxies available") from exc
+
+
+def is_ready() -> bool:
+    return len(_proxy_list) > 0
+
+
+def set_proxy_event():
+    _event.set()
+
+
+# can be generalized and moved to separate file
+def _every(delay: int, task: Callable):
+    global _event
+    next_time = time.time() + delay
+    while not globals.stop_threads:
+        _event.wait(max(0, next_time - time.time()))
+        if not globals.stop_threads:
+            try:
+                task()
+            except Exception as e:
+                logger.error("Problem while fetching proxies")
+                logger.debug(e)
+        # skip tasks if we are behind schedule:
+        next_time += (time.time() - next_time) // delay * delay + delay
+    logger.info("Proxies fetching thread has stopped.")
+
+
+def checker(proxy: str):
+    global _proxy_list
+    ip, port = proxy.split(":")
+    sen = struct.pack("BBB", 0x05, 0x01, 0x00)
+
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.settimeout(5)
+        try:
+            s.connect((ip, int(port)))
+            s.sendall(sen)
+
+            data = s.recv(2)
+            version, auth = struct.unpack("BB", data)
+            # Check if the proxy is socks5 and it doesn't require authentication
+            if version == 5 and auth == 0:
+                _proxy_list[proxy] = proxy
+            else:
+                _proxy_list.pop(proxy, None)
+
+        except Exception as e:
+            logger.info(f"Proxy {proxy} invalid. Reason: {e}")
+            _proxy_list.pop(proxy, None)
+            return