From b2599c3b36ad6d586bcc63c642e9f3d83520fb58 Mon Sep 17 00:00:00 2001 From: micheal Date: Fri, 20 Dec 2024 17:51:38 +1300 Subject: [PATCH] 5.13.2 release, improve smart route. --- code/default/gae_proxy/local/ip_range.txt | 352 +++++++++++++--- .../lib/noarch/front_base/connect_manager.py | 15 +- .../lib/noarch/front_base/http2_stream.py | 9 +- .../lib/noarch/front_base/http_common.py | 4 +- .../lib/noarch/front_base/ip_manager.py | 11 +- code/default/smart_router/local/dns_query.py | 17 +- code/default/smart_router/local/dns_server.py | 3 + .../smart_router/local/proxy_handler.py | 50 ++- .../smart_router/tests/test_dns_query.py | 59 ++- code/default/version.txt | 2 +- code/default/x_tunnel/local/base_container.py | 51 ++- code/default/x_tunnel/local/client.py | 2 +- .../local/cloudflare_front/ip_manager.py | 8 +- code/default/x_tunnel/local/config.py | 20 +- .../x_tunnel/local/front_dispatcher.py | 6 + code/default/x_tunnel/local/global_var.py | 2 +- code/default/x_tunnel/local/proxy_session.py | 393 +++++++++++++----- .../x_tunnel/local/seley_front/config.py | 2 +- .../local/seley_front/connect_creator.py | 5 +- .../x_tunnel/local/seley_front/ip_manager.py | 20 +- .../local/tls_relay_front/ip_manager.py | 16 +- code/default/x_tunnel/local/upload_logs.py | 2 +- 22 files changed, 838 insertions(+), 211 deletions(-) diff --git a/code/default/gae_proxy/local/ip_range.txt b/code/default/gae_proxy/local/ip_range.txt index 6baf3079b8..eef7ad1d38 100644 --- a/code/default/gae_proxy/local/ip_range.txt +++ b/code/default/gae_proxy/local/ip_range.txt @@ -1,57 +1,295 @@ -64.233.160.0-64.233.171.255 -64.233.176.0-64.233.191.255 -66.102.1.0-66.102.1.255 -74.125.20.0-74.125.21.255 -74.125.23.0-74.125.24.255 -74.125.26.0-74.125.26.255 -74.125.28.0-74.125.28.255 -74.125.30.0-74.125.31.255 -74.125.68.0-74.125.71.255 -74.125.90.0-74.125.90.255 -74.125.124.0-74.125.124.255 -74.125.126.0-74.125.136.255 -74.125.138.0-74.125.143.255 -74.125.192.0-74.125.193.255 -74.125.195.0-74.125.197.255 -74.125.199.0-74.125.206.255 -103.47.153.0-103.47.153.255 -104.199.185.0-104.199.185.255 -108.177.8.0-108.177.15.255 -108.177.96.0-108.177.98.255 -108.177.103.0-108.177.104.255 -108.177.111.0-108.177.112.255 -108.177.119.0-108.177.122.255 -108.177.125.0-108.177.127.255 -142.250.4.0-142.250.4.255 -142.250.8.0-142.250.9.255 -172.217.0.0-172.217.31.255 -172.217.160.0-172.217.172.255 -172.217.174.0-172.217.175.255 -172.217.192.0-172.217.195.255 -172.217.197.0-172.217.197.255 -172.217.203.0-172.217.204.255 -172.217.212.0-172.217.212.255 -172.217.214.0-172.217.215.255 -172.217.218.0-172.217.219.255 -172.217.222.0-172.217.222.255 -172.253.58.0-172.253.58.255 -172.253.62.0-172.253.62.255 -172.253.112.0-172.253.113.255 -172.253.115.0-172.253.118.255 -172.253.120.0-172.253.120.255 -172.253.122.0-172.253.123.255 -172.253.126.0-172.253.127.255 -173.194.66.0-173.194.70.255 -173.194.73.0-173.194.74.255 -173.194.76.0-173.194.79.255 -173.194.175.0-173.194.175.255 -173.194.192.0-173.194.208.255 -173.194.210.0-173.194.223.255 -209.85.144.0-209.85.147.255 -209.85.200.0-209.85.203.255 -209.85.232.0-209.85.235.255 -216.58.192.0-216.58.215.255 -216.58.217.0-216.58.217.255 -216.58.220.0-216.58.223.255 -216.239.36.0-216.239.36.255 -216.239.38.0-216.239.38.255 +8.8.4.0/24 +8.8.8.0/24 +8.35.200.0/21 +34.0.228.0/22 +34.0.232.0/21 +34.1.64.0/18 +34.2.32.0/19 +34.2.64.0/18 +34.2.128.0/17 +34.3.0.0/23 +34.3.3.0/24 +34.3.4.0/24 +34.3.8.0/21 +34.3.16.0/20 +34.3.32.0/19 +34.3.64.0/18 +34.4.0.0/14 +34.13.64.0/22 +34.13.72.0/21 +34.13.80.0/20 +34.13.96.0/19 +34.14.128.0/17 +34.15.0.0/16 +34.34.192.0/20 +34.34.208.0/21 +34.34.224.0/19 +34.43.0.0/16 +34.50.128.0/19 +34.52.0.0/17 +34.64.0.0/19 +34.96.0.0/18 +34.98.0.0/18 +34.98.136.0/21 +34.98.144.0/20 +34.98.160.0/19 +34.98.192.0/18 +34.99.0.0/16 +34.100.0.0/17 +34.101.0.0/20 +34.101.16.0/23 +34.101.19.0/24 +34.101.28.0/22 +34.103.0.0/16 +34.104.0.0/20 +34.104.16.0/21 +34.104.24.0/23 +34.104.26.0/24 +34.104.28.0/22 +34.104.32.0/20 +34.104.48.0/24 +34.104.53.0/24 +34.104.54.0/23 +34.109.0.0/16 +34.110.0.0/17 +34.113.0.0/16 +34.114.0.0/15 +34.116.8.0/21 +34.116.16.0/20 +34.116.32.0/19 +34.118.208.0/20 +34.118.224.0/20 +34.119.0.0/16 +34.124.64.0/19 +34.124.96.0/20 +34.126.0.0/18 +34.126.224.0/19 +34.127.128.0/19 +34.127.160.0/20 +34.127.176.0/24 +34.127.181.0/24 +34.127.182.0/23 +34.127.192.0/18 +34.128.0.0/19 +34.128.38.0/23 +34.128.40.0/23 +34.128.50.0/23 +34.128.56.0/23 +34.128.192.0/18 +34.143.0.0/17 +34.144.0.0/17 +34.144.128.0/18 +34.152.70.0/23 +34.152.82.0/23 +34.152.88.0/21 +34.152.96.0/19 +34.152.128.0/17 +34.153.0.0/19 +34.153.34.0/23 +34.153.36.0/23 +34.153.39.0/24 +34.153.56.0/23 +34.153.60.0/22 +34.153.64.0/18 +34.153.192.0/19 +34.153.226.0/23 +34.153.228.0/23 +34.153.231.0/24 +34.153.248.0/23 +34.153.252.0/22 +34.156.0.0/16 +34.157.10.0/23 +34.157.86.0/24 +34.157.120.0/24 +34.157.122.0/23 +34.157.138.0/23 +34.157.214.0/24 +34.157.218.0/23 +34.157.248.0/24 +34.158.0.0/16 +34.167.0.0/16 +34.177.0.0/19 +34.177.38.0/23 +34.177.56.0/21 +34.177.64.0/18 +34.177.128.0/17 +34.178.0.0/16 +34.180.0.0/16 +34.183.0.0/16 +34.184.0.0/13 +35.187.128.0/20 +35.190.96.0/20 +35.190.240.0/20 +35.191.0.0/16 +35.199.128.0/20 +35.201.32.0/21 +35.201.40.0/24 +35.201.42.0/23 +35.201.44.0/22 +35.201.48.0/20 +35.203.192.0/20 +35.203.208.0/23 +35.203.220.0/22 +35.203.224.0/21 +35.203.240.0/20 +35.206.0.0/21 +35.206.8.0/23 +35.206.12.0/22 +35.206.16.0/20 +35.218.0.0/16 +35.219.192.0/19 +35.220.28.0/23 +35.220.30.0/24 +35.229.0.0/20 +35.230.192.0/19 +35.230.224.0/20 +35.235.128.0/18 +35.235.192.0/20 +35.235.208.0/21 +35.235.224.0/19 +35.242.28.0/23 +35.242.30.0/24 +35.243.16.0/20 +35.243.48.0/21 +57.140.192.0/18 +64.15.112.0/20 +64.233.160.0/19 +66.22.228.0/23 +66.102.0.0/20 +66.249.64.0/19 +70.32.128.0/19 +72.14.192.0/18 +74.125.0.0/16 +104.154.0.0/20 +104.154.112.0/24 +104.154.122.0/23 +104.154.124.0/22 +104.155.240.0/20 +104.196.64.0/24 +104.196.72.0/21 +104.196.80.0/20 +104.199.64.0/23 +104.199.240.0/23 +104.237.160.0/19 +107.178.192.0/20 +107.178.224.0/20 +108.170.192.0/18 +108.177.0.0/17 +130.211.0.0/22 +136.22.160.0/20 +136.22.176.0/21 +136.22.184.0/23 +136.22.186.0/24 +136.124.0.0/15 +142.250.0.0/15 +146.148.0.0/23 +152.65.208.0/22 +152.65.214.0/23 +152.65.218.0/23 +152.65.222.0/23 +152.65.224.0/19 +162.120.128.0/17 +172.110.32.0/21 +172.217.0.0/16 +172.253.0.0/16 +173.194.0.0/16 +192.178.0.0/15 +193.186.4.0/24 +199.36.154.0/23 +199.36.156.0/24 +199.192.112.0/23 +199.192.114.0/24 +199.223.237.0/24 +199.223.238.0/23 +207.223.160.0/20 +208.65.152.0/22 +208.68.108.0/22 +208.81.188.0/22 +208.117.224.0/19 +209.85.128.0/17 +216.58.192.0/19 +216.73.80.0/20 +216.239.32.0/19 +2001:4860::/32 +2404:6800::/32 +2404:f340::/32 +2600:1900::/34 +2600:1900:4100::/43 +2600:1900:4130::/44 +2600:1900:4190::/44 +2600:1900:41f0::/44 +2600:1900:4200::/41 +2600:1900:42b0::/44 +2600:1900:42c0::/42 +2600:1900:4300::/40 +2600:1900:4400::/38 +2600:1900:4800::/37 +2600:1900:5000::/38 +2600:1900:5410::/44 +2600:1900:5420::/43 +2600:1900:5440::/42 +2600:1900:5480::/41 +2600:1900:5500::/40 +2600:1900:5600::/39 +2600:1900:5800::/37 +2600:1900:6000::/35 +2600:1900:8010::/44 +2600:1900:8020::/43 +2600:1900:8040::/42 +2600:1900:8080::/41 +2600:1900:8100::/40 +2600:1900:8200::/39 +2600:1900:8400::/38 +2600:1900:8800::/37 +2600:1900:9000::/36 +2600:1900:a000::/35 +2600:1900:c000::/34 +2600:1901:1::/48 +2600:1901:2::/47 +2600:1901:4::/46 +2600:1901:8::/45 +2600:1901:10::/44 +2600:1901:20::/43 +2600:1901:40::/42 +2600:1901:80::/41 +2600:1901:100::/40 +2600:1901:200::/39 +2600:1901:400::/38 +2600:1901:800::/37 +2600:1901:1000::/36 +2600:1901:2000::/35 +2600:1901:4000::/44 +2600:1901:4020::/43 +2600:1901:4040::/42 +2600:1901:4080::/41 +2600:1901:4100::/40 +2600:1901:4200::/39 +2600:1901:4400::/38 +2600:1901:4800::/37 +2600:1901:5000::/36 +2600:1901:6000::/35 +2600:1901:8000::/40 +2600:1901:8190::/44 +2600:1901:81a0::/44 +2600:1901:81d0::/44 +2600:1901:81e0::/44 +2600:1901:8200::/39 +2600:1901:8400::/38 +2600:1901:8800::/37 +2600:1901:9000::/36 +2600:1901:a000::/35 +2600:1901:c000::/34 +2600:1902::/31 +2600:1904::/30 +2600:1908::/29 +2605:ef80::/32 +2606:40::/32 +2606:73c0::/32 +2607:1c0:241:40::/60 +2607:1c0:300::/40 +2607:f8b0::/32 +2620:11a:a000::/40 +2620:120:e000::/40 +2800:3f0::/32 +2a00:1450::/32 +2c0f:fb50::/32 \ No newline at end of file diff --git a/code/default/lib/noarch/front_base/connect_manager.py b/code/default/lib/noarch/front_base/connect_manager.py index 559ae11bd6..0b3ab3a955 100644 --- a/code/default/lib/noarch/front_base/connect_manager.py +++ b/code/default/lib/noarch/front_base/connect_manager.py @@ -278,8 +278,8 @@ def _connect_thread(self, sleep_time=0): def _connect_process(self): try: - ip_str, sni, host = self.ip_manager.get_ip_sni_host() - if not ip_str: + host_info = self.ip_manager.get_ip_sni_host() + if not host_info: self.no_ip_time = time.time() with self.no_ip_lock: # self.logger.warning("not enough ip") @@ -287,7 +287,7 @@ def _connect_process(self): return None # self.logger.debug("create ssl conn %s", ip_str) - ssl_sock = self._create_ssl_connection(ip_str, sni, host) + ssl_sock = self._create_ssl_connection(host_info) if not ssl_sock: time.sleep(1) return None @@ -310,10 +310,12 @@ def _connect_ssl(self, ip_str, sni, host, close_cb, queue): self.logger.warn("_connect_ssl %s sni:%s host:%s fail:%r", ip_str, sni, host, e) queue.put(e) - def _create_ssl_connection(self, ip_str, sni, host): - try: - # ssl_sock = self.connect_creator.connect_ssl(ip_str, sni, host, close_cb=self.ip_manager.ssl_closed) + def _create_ssl_connection(self, host_info): + ip_str = host_info["ip_str"] + sni = host_info["sni"] + host = host_info["host"] + try: q = Queue() fn_args = (ip_str, sni, host, self.ip_manager.ssl_closed, q) t = threading.Thread(target=self._connect_ssl, args=fn_args, name="connect_ssl_%s" % ip_str) @@ -330,6 +332,7 @@ def _create_ssl_connection(self, ip_str, sni, host): self.ip_manager.update_ip(ip_str, sni, ssl_sock.handshake_time) self.logger.debug("create_ssl update ip:%s time:%d h2:%d sni:%s, host:%s", ip_str, ssl_sock.handshake_time, ssl_sock.h2, ssl_sock.sni, ssl_sock.host) + ssl_sock.host_info = host_info return ssl_sock except socket.error as e: diff --git a/code/default/lib/noarch/front_base/http2_stream.py b/code/default/lib/noarch/front_base/http2_stream.py index e8245e9949..2610eb3ed3 100644 --- a/code/default/lib/noarch/front_base/http2_stream.py +++ b/code/default/lib/noarch/front_base/http2_stream.py @@ -271,9 +271,6 @@ def receive_frame(self, frame): pass if b'END_HEADERS' in frame.flags: - if self.config.http2_show_debug: - self.logger.debug("END_HEADERS") - if self.response_headers is not None: raise ProtocolError("Too many header blocks.") @@ -307,8 +304,6 @@ def receive_frame(self, frame): self.send_response() if b'END_STREAM' in frame.flags: - if self.config.http2_show_debug: - self.logger.debug("%s Closing remote side of stream:%d", self.connection.ssl_sock.ip_str, self.stream_id) xcost = self.response_headers.get("X-Cost", -1) if isinstance(xcost, list): @@ -319,6 +314,10 @@ def receive_frame(self, frame): rtt = whole_cost - xcost receive_cost = time_now - self.get_head_time bytes_received = self.connection._sock.bytes_received - self.start_connection_point + if self.config.http2_show_debug: + self.logger.debug("%s stream:%d END_STREAM %s%s", self.connection.ssl_sock.ip_str, + self.stream_id, self.task.host, self.task.path) + if b"ping" in self.task.path and self.config.http2_show_debug: self.logger.debug("got pong for %s", self.connection.ip_str) diff --git a/code/default/lib/noarch/front_base/http_common.py b/code/default/lib/noarch/front_base/http_common.py index bb946183d0..4e6a6c1195 100644 --- a/code/default/lib/noarch/front_base/http_common.py +++ b/code/default/lib/noarch/front_base/http_common.py @@ -199,6 +199,7 @@ def __init__(self, logger, ip_manager, config, ssl_sock, close_cb, retry_task_cb self.ssl_sock = ssl_sock self.handshake = ssl_sock.handshake_time * 0.001 self.rtt = ssl_sock.handshake_time * 0.001 + self.adjust = float(ssl_sock.host_info.get("adjust", 0)) self.streams = {} self.ip_str = ssl_sock.ip_str self.close_cb = close_cb @@ -226,6 +227,7 @@ def __str__(self): o += " processed_tasks: %d\r\n" % (self.processed_tasks) o += " continue_fail_tasks: %s\r\n" % (self.continue_fail_tasks) o += " handshake: %f \r\n" % self.handshake + o += " adjust: %f \r\n" % self.adjust o += " rtt_history: %s\r\n" % (self.rtt_history) o += " adjust_history: %s\r\n" % (self.adjust_history) if self.version != "1.1": @@ -301,7 +303,7 @@ def get_score(self): self.logger.debug("get_score %s, speed:%f rtt:%d stream_num:%d score:%f", self.ip_str, speed * 0.000001, self.rtt * 1000, len(self.streams), score) - return score + return score + self.adjust def get_host(self, task_host): if task_host: diff --git a/code/default/lib/noarch/front_base/ip_manager.py b/code/default/lib/noarch/front_base/ip_manager.py index 16a50c039c..034c676cbf 100644 --- a/code/default/lib/noarch/front_base/ip_manager.py +++ b/code/default/lib/noarch/front_base/ip_manager.py @@ -405,7 +405,7 @@ def get_ip_sni_host(self, to_recheck=False): if ip_num == 0: # self.logger.warning("no ip") time.sleep(5) - return None, None, None + return None ip_connect_interval = ip_num * self.scan_recheck_interval + 200 if to_recheck else self.ip_connect_interval @@ -467,7 +467,11 @@ def get_ip_sni_host(self, to_recheck=False): self.ip_pointer += 1 sni, host = self.host_manager.get_sni_host(ip_str) - return ip_str, sni, host + return { + "ip_str": ip_str, + "sni": sni, + "host": host, + } except Exception as e: self.logger.exception("get_ip fail:%r", e) finally: @@ -786,7 +790,8 @@ def scan_ip_worker(self): self.check_local_network.is_ok(): if self.good_ip_num >= self.max_good_ip_num * 0.6 and \ len(self.ip_list) >= self.max_good_ip_num * 0.9: - ip_str, sni, host = self.get_ip_sni_host() + host_info = self.get_ip_sni_host() + ip_str = host_info["ip_str"] if ip_str and self.check_local_network.is_ok(ip_str): self.recheck_ip(ip_str, first_report=False) time.sleep(self.scan_recheck_interval) diff --git a/code/default/smart_router/local/dns_query.py b/code/default/smart_router/local/dns_query.py index f8c4d1055c..2bcc3a1caf 100644 --- a/code/default/smart_router/local/dns_query.py +++ b/code/default/smart_router/local/dns_query.py @@ -482,7 +482,11 @@ def __init__(self, timeout=6): self.protocol = "DoH" self.timeout = timeout self.cn_servers = ["https://1.12.12.12/dns-query", "https://223.5.5.5/dns-query"] - self.other_servers = ["https://1.1.1.1/dns-query"] + self.other_servers = [ + "https://1.1.1.1/dns-query", + "https://dns10.quad9.net/dns-query", + "https://dns.aa.net.uk/dns-query", + ] self.connection_timeout = 60 self.connections = [] @@ -508,10 +512,7 @@ def get_connection(self): @property def server(self): - if g.config.country_code == "CN": - return random.choice(self.cn_servers) - else: - return random.choice(self.other_servers) + return random.choice(self.other_servers) def query_json(self, domain, dns_type=1): try: @@ -540,12 +541,14 @@ def query_json(self, domain, dns_type=1): xlog.warn("DNSOverHttpsQuery query fail:%r", e) return [] - def query(self, domain, dns_type=1): + def query(self, domain, dns_type=1, url=None): t0 = time.time() try: client = self.get_connection() - url = self.server + if not url: + url = self.server + # xlog.debug("DoH use %s", url) d = DNSRecord(DNSHeader()) d.add_question(DNSQuestion(domain, dns_type)) diff --git a/code/default/smart_router/local/dns_server.py b/code/default/smart_router/local/dns_server.py index 89d97eddad..4427744add 100644 --- a/code/default/smart_router/local/dns_server.py +++ b/code/default/smart_router/local/dns_server.py @@ -185,6 +185,9 @@ def on_udp_relay(self, rsock, req_data, from_addr): return xlog.debug("UDP relay from %s size:%d to:%s:%d", from_addr, len(data), addr, port) + if port != 53: + return + head_length = len(req_data) - len(data) head = req_data[:head_length] res_data = self.dns_query(data, from_addr) diff --git a/code/default/smart_router/local/proxy_handler.py b/code/default/smart_router/local/proxy_handler.py index 6580c80642..e8913439c0 100644 --- a/code/default/smart_router/local/proxy_handler.py +++ b/code/default/smart_router/local/proxy_handler.py @@ -1,9 +1,11 @@ import time import socket import struct +import json try: from urllib.parse import urlparse + from urllib.parse import parse_qs except ImportError: from urlparse import urlparse @@ -354,13 +356,53 @@ def http_handler(self): url_prex_len = len(url) path = b"/" else: - # not proxy request, should be PAC - xlog.debug("PAC %s %s from:%s", method, url, self.client_address) - handler = pac_server.PacHandler(self.conn, self.client_address, None, xlog) - return handler.handle() + # not proxy request + parsed_url = urlparse(utils.to_str(url)) + kv = parse_qs(parsed_url.query) + if parsed_url.path == "/dns-query": + return self.DoH_handler(kv) + else: + xlog.debug("PAC %s %s from:%s", method, url, self.client_address) + handler = pac_server.PacHandler(self.conn, self.client_address, None, xlog) + return handler.handle() sock = SocketWrap(self.conn, self.client_address[0], self.client_address[1]) sock.replace_pattern = [url[:url_prex_len], b""] xlog.debug("http %r connect to %s:%d %s %s", self.client_address, host, port, method, path) handle_domain_proxy(sock, host, port, self.client_address) + + def DoH_handler(self, kv): + handler = pac_server.PacHandler(self.conn, self.client_address, None, xlog) + name = kv.get("name", [None])[0] + if not name: + xlog.warn("DoH request no name") + return handler.send_response(content=b'{"error":"no name"}', status=400) + + dns_type = kv.get("type", ["1"])[0] + if dns_type.isnumeric(): + dns_type = int(dns_type) + + ips = utils.to_str(g.dns_query.query(name, dns_type)) + info = { + "Status": 0, + "Answer": [ + ] + } + for ip in ips: + if dns_type == 1 and not utils.check_ip_valid4(ip): + continue + if dns_type == 16 and not utils.check_ip_valid6(ip): + continue + + info["Answer"].append({ + "name": name, + "type": dns_type, + "data": ip + }) + res = json.dumps(info) + headers = { + "Content-Type": "application/dns-json", + "Access-Control-Allow-Origin": "*" + } + return handler.send_response(content=res, headers=headers, status=200) diff --git a/code/default/smart_router/tests/test_dns_query.py b/code/default/smart_router/tests/test_dns_query.py index 54d94f3aad..ae8699928a 100644 --- a/code/default/smart_router/tests/test_dns_query.py +++ b/code/default/smart_router/tests/test_dns_query.py @@ -1,12 +1,28 @@ - +import json +import time from unittest import TestCase +import os +import sys + +import requests + +current_path = os.path.dirname(os.path.abspath(__file__)) +default_path = os.path.abspath(os.path.join(current_path, os.pardir, os.pardir)) +noarch_path = os.path.abspath(os.path.join(default_path, 'lib', "noarch")) +sys.path.append(noarch_path) +sys.path.append(default_path) from dnslib.dns import DNSRecord, DNSQuestion, QTYPE -from smart_router.local.dns_query import LocalDnsQuery +from smart_router.local.dns_query import LocalDnsQuery, DnsOverHttpsQuery, g +class MockConfig(object): + def __init__(self): + self.PROXY_ENABLE = False + class TestDnsQuery(TestCase): def test_local_udp_query(self): + qr = LocalDnsQuery() ips = qr.query('www.microsoft.com', timeout=1000) self.assertTrue(len(ips) > 0) @@ -17,3 +33,42 @@ def test_local_udp_query(self): # a_pkt = query.send("127.0.0.1", 53, tcp=False, timeout=5) # a = DNSRecord.parse(a_pkt) # print(a) + + def test_DoH_json_query(self): + servers = [ + # "https://1.1.1.1/dns-query", + # "https://dns10.quad9.net/dns-query", + # "https://dns.aa.net.uk/dns-query", + "https://doh.la.ahadns.net/dns-query" + ] + domain = "vs6.85po.com" + for server in servers: + url = server + "?name=" + domain + "&type=A" # type need to map to Text. + r = requests.request("GET", url, headers={"accept": "application/dns-json"}) + ips = [] + if not r: + print(f"{server} failed") + continue + + t = r.text.encode("utf-8") + + data = json.loads(t) + for answer in data["Answer"]: + ips.append(answer["data"]) + + print(f"server:{server} ips: {ips}") + + def test_DoH_query(self): + g.config = MockConfig() + qr = DnsOverHttpsQuery() + domain = "vs6.85po.com" + for url in [ + "https://1.1.1.1/dns-query", + "https://dns10.quad9.net/dns-query", + "https://dns.aa.net.uk/dns-query", + "https://freedns.controld.com/p0" + ]: + t0 = time.time() + ips = qr.query(domain, url=url) + t1 = time.time() + print(f"use {url} ips:{ips} cost:{t1-t0}") diff --git a/code/default/version.txt b/code/default/version.txt index b54822f0da..0b17707f79 100644 --- a/code/default/version.txt +++ b/code/default/version.txt @@ -1 +1 @@ -5.10.7 \ No newline at end of file +5.13.2 \ No newline at end of file diff --git a/code/default/x_tunnel/local/base_container.py b/code/default/x_tunnel/local/base_container.py index 005761ea1e..b4c548c06d 100644 --- a/code/default/x_tunnel/local/base_container.py +++ b/code/default/x_tunnel/local/base_container.py @@ -284,19 +284,25 @@ def status(self): class BlockReceivePool(): - def __init__(self, process_callback): + def __init__(self, process_callback, logger): self.lock = threading.Lock() self.process_callback = process_callback + self.logger = logger self.reset() def reset(self): # xlog.info("recv_pool reset") self.next_sn = 1 self.block_list = [] + self.timeout_sn_list = {} def put(self, sn, data): + # xlog.debug("recv_pool put sn:%d len:%d", sn, len(data)) self.lock.acquire() try: + if sn in self.timeout_sn_list: + del self.timeout_sn_list[sn] + if sn < self.next_sn: # xlog.warn("recv_pool put timeout sn:%d", sn) return False @@ -324,6 +330,44 @@ def put(self, sn, data): finally: self.lock.release() + def mark_sn_timeout(self, sn, t, server_time): + # xlog.warn("mark_sn_timeout down_sn:%d", sn) + with self.lock: + if sn not in self.timeout_sn_list: + self.logger.warn("mark_sn_timeout sn:%d t:%f", sn, server_time - t) + self.timeout_sn_list[sn] = { + "server_send_time": t, + } + elif t > self.timeout_sn_list[sn]["server_send_time"]: + self.logger.warn("mark_sn_timeout renew sn:%d t:%f", sn, server_time - t) + self.timeout_sn_list[sn]["server_send_time"] = t + + def get_timeout_list(self, server_time, timeout): + sn_list = [] + with self.lock: + for sn, info in self.timeout_sn_list.items(): + if server_time - info["server_send_time"] < timeout: + continue + + if server_time - info.get("retry_time", server_time) < timeout: + continue + + self.logger.warn("get_timeout_list sn:%d sent:%f retry:%f", sn, server_time - info["server_send_time"], + server_time - info.get("retry_time", server_time)) + info["retry_time"] = server_time + sn_list.append(sn) + + return sn_list + + def is_received(self, sn): + if sn < self.next_sn: + return True + + if sn in self.block_list: + return True + + return False + def status(self): out_string = "Block_receive_pool:\r\n" out_string += " next_sn:%d\r\n" % self.next_sn @@ -600,8 +644,8 @@ def stop(self, reason=""): name="do_stop_%s:%d" % (self.host, self.port)).start() def do_stop(self, reason="unknown"): - self.xlog.debug("Conn session:%s conn:%d fd:%d stop:%s", utils.to_str(self.session.session_id), self.conn_id, - self._fd, reason) + self.xlog.debug("Conn session:%s %s:%d conn:%d fd:%d stop:%s", utils.to_str(self.session.session_id), + self.host, self.port, self.conn_id, self._fd, reason) self.running = False self.connection_pipe.remove_sock(self.sock) @@ -782,6 +826,7 @@ def send_to_sock(self, data): except Exception as e: self.xlog.info("%s conn:%d send closed: %r", self.session.session_id, self.conn_id, e) if self.is_client: + self.transfer_peer_close("send fail.") self.do_stop(reason="send fail.") return False diff --git a/code/default/x_tunnel/local/client.py b/code/default/x_tunnel/local/client.py index f95caecca4..c6f267b9c8 100644 --- a/code/default/x_tunnel/local/client.py +++ b/code/default/x_tunnel/local/client.py @@ -62,7 +62,7 @@ def xxnet_version(): def get_launcher_uuid(): launcher_config_fn = os.path.join(data_path, "launcher", "config.json") try: - with open(launcher_config_fn, "r") as fd: + with open(launcher_config_fn, "r", encoding='utf-8') as fd: info = json.load(fd) return info.get("update_uuid") except Exception as e: diff --git a/code/default/x_tunnel/local/cloudflare_front/ip_manager.py b/code/default/x_tunnel/local/cloudflare_front/ip_manager.py index 8fd3adb8cd..03f2af7948 100644 --- a/code/default/x_tunnel/local/cloudflare_front/ip_manager.py +++ b/code/default/x_tunnel/local/cloudflare_front/ip_manager.py @@ -85,9 +85,13 @@ def get_ip_sni_host(self): self.logger.debug("get ip:%s sni:%s", ip, sni) info["links"] += 1 info["last_try"] = now - return ip, sni, top_domain + return { + "ip_str": ip, + "sni": sni, + "host": top_domain, + } - return None, None, None + return None def _get_domain(self, top_domain): self.domain_map.setdefault(top_domain, { diff --git a/code/default/x_tunnel/local/config.py b/code/default/x_tunnel/local/config.py index f854028c05..8dac0d5d01 100644 --- a/code/default/x_tunnel/local/config.py +++ b/code/default/x_tunnel/local/config.py @@ -55,6 +55,12 @@ def load_config(): # min roundtrip on road if connectoin exist config.set_var("min_on_road", 3) + config.set_var("server_time_max_deviation", 0.6) + + config.set_var("send_timeout_retry", 4) + + config.set_var("server_download_timeout_retry", 4) + # range 1 - 1000, ms config.set_var("send_delay", 10) @@ -84,6 +90,7 @@ def load_config(): config.set_var("enable_seley", 1) config.set_var("enable_tls_relay", 1) config.set_var("enable_direct", 0) + config.set_var("local_auto_front", 1) config.load() @@ -91,12 +98,13 @@ def load_config(): config.windows_size = config.max_payload * config.concurent_thread_num * 2 xlog.info("X-Tunnel window:%d", config.windows_size) - if "localhost" in config.server_host or "127.0.0.1" in config.server_host: - config.enable_cloudflare = 0 - config.enable_tls_relay = 0 - config.enable_seley = 0 - config.enable_direct = 1 - xlog.info("Only enable Direct front for localhost") + if config.local_auto_front: + if "localhost" in config.server_host or "127.0.0.1" in config.server_host: + config.enable_cloudflare = 0 + config.enable_tls_relay = 0 + config.enable_seley = 0 + config.enable_direct = 1 + xlog.info("Only enable Direct front for localhost") if config.write_log_file: xlog.log_to_file(os.path.join(data_path, "client.log")) diff --git a/code/default/x_tunnel/local/front_dispatcher.py b/code/default/x_tunnel/local/front_dispatcher.py index 6ed2807b56..a326f84e92 100644 --- a/code/default/x_tunnel/local/front_dispatcher.py +++ b/code/default/x_tunnel/local/front_dispatcher.py @@ -157,11 +157,17 @@ def request(method, host, path="/", headers={}, data="", timeout=100): content, status, response = "", 603, {} while time.time() - start_time < timeout: + start_get_front = time.time() front = get_front(host, timeout) if not front: xlog.warn("get_front fail") return "", 602, {} + finished_get_front = time.time() + get_front_time = finished_get_front - start_get_front + if get_front_time > 0.1: + xlog.warn("get_front_time: %f for %s %s %s", get_front_time, method, host, path) + if host == "dns.xx-net.org" and front == cloudflare_front and g.server_host: # share the x-tunnel connection with dns.xx-net.org # x-tunnel server will forward the request to dns.xx-net.org diff --git a/code/default/x_tunnel/local/global_var.py b/code/default/x_tunnel/local/global_var.py index 8a581d5347..7c4d80af1a 100644 --- a/code/default/x_tunnel/local/global_var.py +++ b/code/default/x_tunnel/local/global_var.py @@ -3,7 +3,7 @@ system = "" running = True -protocol_version = 2 +protocol_version = 4 last_refresh_time = 0 login_process = False data_path = None diff --git a/code/default/x_tunnel/local/proxy_session.py b/code/default/x_tunnel/local/proxy_session.py index a657416fd9..783fa63ae2 100644 --- a/code/default/x_tunnel/local/proxy_session.py +++ b/code/default/x_tunnel/local/proxy_session.py @@ -60,7 +60,7 @@ def __init__(self): self.config = g.config self.wait_queue = base_container.WaitQueue() self.send_buffer = base_container.SendBuffer(max_payload=g.config.max_payload) - self.receive_process = base_container.BlockReceivePool(self.download_data_processor) + self.receive_process = base_container.BlockReceivePool(self.download_data_processor, xlog) self.connection_pipe = base_container.ConnectionPipe(self, xlog) self.lock = threading.Lock() # lock for conn_id, sn generation, on_road_num change, @@ -90,6 +90,10 @@ def __init__(self): self.upload_speed = 0.0 self.download_speed = 0.0 + # server time logic like NTP + self.server_time_offset = 0 + self.server_time_deviation = 9999 + # the receive time of the tail of the socket receive buffer # if now - oldest_received_time > delay, then send. # set only no data in receive buffer @@ -105,6 +109,9 @@ def __init__(self): if g.config.upload_logs: threading.Thread(target=upload_logs_thread, name="upload_logs").start() + self.timeout_check_th = threading.Thread(target=self.timeout_checker, name="timeout_check") + self.timeout_check_th.start() + def start(self): with self.lock: if self.running is True: @@ -134,6 +141,8 @@ def start(self): self.received_sn = [] self.receive_next_sn = 1 self.target_on_roads = 0 + self.server_time_offset = 0 + self.server_time_deviation = 9999 if not self.login_session(): xlog.warn("x-tunnel login_session fail, session not start") @@ -147,13 +156,30 @@ def start(self): self.round_trip_thread[i] = threading.Thread(target=self.normal_round_trip_worker, args=(i,), name="roundtrip_%d" % i) - self.round_trip_thread[i].daemon = True self.round_trip_thread[i].start() self.connection_pipe.start() xlog.info("session started.") return True + def timeout_checker(self): + while self.running: + timeout_num = 0 + with self.lock: + time_now = time.time() + for sn, data_info in self.transfer_list.items(): + if data_info["stat"] != "timeout" and time_now - (data_info["start_time"] + data_info["server_timeout"]) > g.config.send_timeout_retry: + data_info["stat"] = "timeout" + xlog.warn("timeout_checker found transfer_no:%d timeout:%f", sn, time_now - data_info["start_time"]) + timeout_num += 1 + + if timeout_num: + self.target_on_roads = \ + min(g.config.concurent_thread_num - g.config.min_on_road, self.target_on_roads + timeout_num) + self.trigger_more() + + time.sleep(1) + def traffic_speed_calculation(self): now = time.time() time_go = now - self.last_traffic_reset_time @@ -175,15 +201,14 @@ def stop(self): # xlog.warn("session stop but not running") return + self.running = False + self.session_id = "" + self.target_on_roads = 0 with self.lock: - self.running = False - self.target_on_roads = 0 for i in range(0, g.config.concurent_thread_num): self.wait_queue.notify() - self.session_id = "" self.close_all_connection() - self.send_buffer.reset() self.receive_process.reset() self.wait_queue.stop() @@ -321,7 +346,8 @@ def get_stat(self, type="second"): def status(self): self.traffic_speed_calculation() - out_string = "session_id: %s\n" % self.session_id + out_string = "session_id: %s\n" % utils.to_str(self.session_id) + out_string += "server: %s\n" % g.server_host out_string += "extra_info: %s\n" % json.dumps(json.loads(self.get_login_extra_info()), indent=2) out_string += "thread num: %d\n" % threading.active_count() out_string += "running: %d\n" % self.running @@ -336,16 +362,15 @@ def status(self): out_string += "upload_speed: %f\n" % self.upload_speed out_string += "download_speed: %f\n" % self.download_speed out_string += "last_traffic_reset_time %f ago\n" % (time.time() - self.last_traffic_reset_time ) - + out_string += "server_time_offset: %f\n" % self.server_time_offset + out_string += "server_time_deviation: %f\n" % self.server_time_deviation + out_string += "target_on_roads: %d\n" % self.target_on_roads out_string += "on_road_num:%d\n" % self.on_road_num out_string += "transfer_list: %d\n" % len(self.transfer_list) - for transfer_no in sorted(self.transfer_list.keys()): - transfer = self.transfer_list[transfer_no] - if "start" in self.transfer_list[transfer_no]: - time_way = " t:" + str((time.time() - self.transfer_list[transfer_no]["start"])) - else: - time_way = "" - out_string += "[%d] %s %s\n" % (transfer_no, json.dumps(transfer), time_way) + for sn in sorted(self.transfer_list.keys()): + data_info = self.transfer_list[sn] + time_way = " t:" + str((time.time() - self.transfer_list[sn]["start_time"])) + out_string += f'[{sn}] stat:{data_info["stat"]} server_timeout:{data_info["server_timeout"]} retry:{data_info["retry"]} {time_way}\n' out_string += "\n" + self.wait_queue.status() out_string += "\n" + self.send_buffer.status() @@ -489,7 +514,7 @@ def remove_conn(self, conn_id): try: if conn_id in self.conn_list: conn = self.conn_list[conn_id] - xlog.debug("remove conn:%d %s:%d", conn_id, conn.host, conn.port) + # xlog.debug("remove conn:%d %s:%d", conn_id, conn.host, conn.port) del self.conn_list[conn_id] except Exception as e: xlog.warn("remove conn:%d except:%r", conn_id, e) @@ -574,24 +599,37 @@ def get_ack(self, force=False): return "" + def get_down_sn_timeout_list_pack(self): + buf = base_container.WriteBuffer() + if self.server_time_deviation > g.config.server_time_max_deviation: + return buf + + server_time = int(time.time() + self.server_time_offset) + timeout_list = self.receive_process.get_timeout_list(server_time, g.config.server_download_timeout_retry) + for sn in timeout_list: + buf.append(struct.pack(" 4 or self.on_road_num < self.target_on_roads: # xlog.debug("got data, force get ack") force = True ack = self.get_ack(force=force) - if data or ack or force: + if force or ack: # xlog.debug("get_send_data work_id:%d data_len:%d ack_len:%d force:%d", work_id, len(data), len(ack), force) - return data, ack + return data, ack, down_sn_timeout_list_pack self.wait_queue.wait(work_id) xlog.debug("get_send_data on stop") - return b"", b"" + return b"", b"", b"" def ack_process(self, ack): self.lock.acquire() @@ -637,16 +675,100 @@ def download_data_processor(self, data): except Exception as e: xlog.exception("download_data_processor:%r", e) - def round_trip_process(self, data, ack): + def check_upload_not_acked(self, server_time): + server_local_time = server_time - self.server_time_offset + if self.server_time_deviation > g.config.server_time_max_deviation: + return + + timeout_num = 0 + entry_time = time.time() + with self.lock: + now = time.time() + if now - entry_time > 0.1: + xlog.error("check_upload_not_acked lock time:%f", now - entry_time) + return + + for no, data_info in self.transfer_list.items(): + if data_info["stat"] == "timeout": + continue + + if data_info["server_received"] == False and server_local_time - data_info["start_time"] > g.config.send_timeout_retry: + data_info["stat"] = "timeout" + xlog.warn("check_upload_not_acked found transfer_no:%d upload timeout:%f", no, + server_local_time - data_info["start_time"]) + timeout_num += 1 + continue + + if data_info["server_sent"] and server_time - data_info["server_sent"] > g.config.send_timeout_retry: + data_info["stat"] = "timeout" + xlog.warn("check_upload_not_acked found transfer_no:%d down timeout:%f", no, + server_time - data_info["server_sent"]) + timeout_num += 1 + continue + + if timeout_num: + self.target_on_roads = \ + min(g.config.concurent_thread_num - g.config.min_on_road, self.target_on_roads + timeout_num) + self.trigger_more() + + def process_server_received_transfer_no(self, server_received_no_list, server_sent_no_list, server_time): + server_received_next_no = struct.unpack(" g.config.server_time_max_deviation: + return + + server_time = time.time() + self.server_time_offset + + sn_num = struct.unpack(" g.config.server_download_timeout_retry: + # xlog.warn("server unacked sn:%d timeout:%f", sn, server_time - t) + self.receive_process.mark_sn_timeout(sn, t, server_time) + timeout_num += 1 + + def round_trip_process(self, data, ack, server_rcvd_no_list, server_sent_no_list, server_unack_snd_sn, server_time): while len(data): sn, plen = struct.unpack(" g.config.max_payload or \ - len(self.wait_queue.waiters) < g.config.min_on_road: + len(self.wait_queue.waiters) < g.config.min_on_road or \ + self.server_time_deviation > g.config.server_time_max_deviation or \ + data_info["stat"] == "retry": # xlog.debug("pool_size:%s waiters:%d", self.send_buffer.pool_size, len(self.wait_queue.waiters)) server_timeout = 0 else: server_timeout = g.config.roundtrip_timeout - request_session_id = self.session_id - upload_data_head = struct.pack(" 0.1: - xlog.warn("lock_time: %f", lock_time) - if g.config.show_debug: self.traffic_speed_calculation() xlog.debug("start trip, tid:%d target:%d running:%d timeout:%d send:%d", transfer_no, self.target_on_roads, self.on_road_num, server_timeout, len(upload_post_data)) - while self.running: + sleep_time = 1 + # Use one time loop for easy quit and clean up. + for _ in range(1): + upload_post_data2 = bytearray(upload_post_data) try: content, status, response = g.http_client.request(method="POST", host=g.server_host, path="/data?tid=%d" % transfer_no, - data=upload_post_data, + data=upload_post_data2, headers={ - "Content-Length": str(len(upload_post_data)), + "Content-Length": str(len(upload_post_data2)), }, timeout=server_timeout + g.config.network_timeout) - traffic = len(upload_post_data) + len(content) + 645 - self.traffic_upload += len(upload_post_data) + 645 + traffic = len(upload_post_data2) + len(content) + 645 + self.traffic_upload += len(upload_post_data2) + 645 self.traffic_download += len(content) g.quota -= traffic if g.quota < 0: @@ -744,31 +923,39 @@ def roundtrip_task(self, work_id): if self.running: xlog.exception("request except:%r ", e) + data_info["stat"] = "timeout" time.sleep(sleep_time) continue + finally: + with self.lock: + self.on_road_num -= 1 g.stat["roundtrip_num"] += 1 - roundtrip_time = (time.time() - start_time) + time_now = time.time() + roundtrip_time = (time_now - start_time) if status == 521: xlog.warn("X-tunnel server is down, try get new server.") g.server_host = None - self.on_road_num -= 1 self.stop() login_process() return if status != 200: - xlog.warn("roundtrip time:%f transfer_no:%d send:%d status:%r ", - roundtrip_time, transfer_no, send_data_len, status) + head = upload_post_data2[:3] + xlog.warn("roundtrip time:%f transfer_no:%d send:%d head:%s status:%r ", + roundtrip_time, transfer_no, send_data_len, utils.str2hex(head), status) + data_info["stat"] = "timeout" time.sleep(sleep_time) continue content_length = int(response.headers.get(b"Content-Length", b"0")) - recv_len = len(content) - if recv_len < 6 or (content_length and content_length != recv_len): + content_len = len(content) + if content_len < 6 or (content_length and content_length != content_len): xlog.warn("roundtrip time:%f transfer_no:%d send:%d recv:%d Head:%d", - roundtrip_time, transfer_no, send_data_len, recv_len, content_length) + roundtrip_time, transfer_no, send_data_len, content_len, content_length) + + data_info["stat"] = "timeout" continue try: @@ -776,8 +963,9 @@ def roundtrip_task(self, work_id): payload = base_container.ReadBuffer(content) magic, version, pack_type = struct.unpack("%f, deviation:%f->%f", self.server_time_offset, new_offset, + self.server_time_deviation, roundtrip_time) + self.server_time_offset = new_offset + self.server_time_deviation = roundtrip_time + xlog.debug( "no:%d %s " "road_time:%f " @@ -839,13 +1029,15 @@ def roundtrip_task(self, work_id): self.target_on_roads, speed, roundtrip_time, server_timeout ) + if g.config.show_debug: + xlog.debug("data:%d ack:%d rcvd_no:%d sent_no:%d unack_sent_no:%d", data_len, ack_len, rcvd_no_len, sent_no_len, unack_snd_sn_len) if len(self.conn_list) == 0: self.target_on_roads = 0 elif len(content) >= g.config.max_payload: self.target_on_roads = \ min(g.config.concurent_thread_num - g.config.min_on_road, self.target_on_roads + 10) - elif len(content) <= 53: + elif data_len <= 200: self.target_on_roads = max(g.config.min_on_road, self.target_on_roads - 5) self.trigger_more() # xlog.debug("target roundtrip: %d, on_road: %d", self.target_on_roads, self.on_road_num) @@ -858,39 +1050,40 @@ def roundtrip_task(self, work_id): data = payload.get_buf(data_len) ack = payload.get_buf(ack_len) - if len(data) != data_len or len(ack) != ack_len: - xlog.warn("left data error:%d data_len:%d/%d ack_len:%d/%d", len(payload), len(data), data_len, - len(ack), ack_len) - continue + rcvd_no_list = payload.get_buf(rcvd_no_len) + sent_no_list = payload.get_buf(sent_no_len) + unack_snd_sn = payload.get_buf(unack_snd_sn_len) + ext = payload.get_buf(ext_len) - if len(payload) >= 32: - checksum_str = utils.to_str(payload.get(32).tobytes()) - checksum = hashlib.md5(bytes(content[:-32])).hexdigest() - if checksum != checksum_str: - xlog.warn("checksum error:%s %s", checksum_str, checksum) - continue + checksum_str = utils.to_str(payload.get(32).tobytes()) + checksum = hashlib.md5(bytes(content[:-32])).hexdigest() + if checksum != checksum_str: + xlog.warn("checksum error:%s %s", checksum_str, checksum) - except Exception as e: - xlog.warn("trip:%d no:%d data not enough %r", work_id, transfer_no, e) - continue + data_info["stat"] = "timeout" + continue - # xlog.debug("trip:%d no:%d recv data:%s", work_id, transfer_no, parse_data(data)) + self.last_receive_time = time.time() - try: - self.round_trip_process(data, ack) + with self.lock: + if transfer_no in self.transfer_list: + del self.transfer_list[transfer_no] - self.last_receive_time = time.time() - break + self.round_trip_process(data, ack, rcvd_no_list, sent_no_list, unack_snd_sn, server_time) + self.check_upload_not_acked(server_time) + + return except Exception as e: - xlog.exception("data process:%r", e) + xlog.exception("trip:%d no:%d data not enough %r", work_id, transfer_no, e) - with self.lock: - self.on_road_num -= 1 - try: - if transfer_no in self.transfer_list: - del self.transfer_list[transfer_no] - except: - pass + data_info["stat"] = "timeout" + continue + + # xlog.debug("trip:%d no:%d recv data:%s", work_id, transfer_no, parse_data(data)) + + xlog.warn("roundtrip failed, no:%d target:%d on_road:%d timeout:%d send:%d", + transfer_no, self.target_on_roads, self.on_road_num, server_timeout, + len(upload_post_data)) def parse_data(data): diff --git a/code/default/x_tunnel/local/seley_front/config.py b/code/default/x_tunnel/local/seley_front/config.py index ba799368cf..1d65775455 100644 --- a/code/default/x_tunnel/local/seley_front/config.py +++ b/code/default/x_tunnel/local/seley_front/config.py @@ -27,7 +27,7 @@ def __init__(self, fn): # connect_manager self.set_var("connection_pool_min", 1) - self.set_var("max_links_per_ip", 17) + self.set_var("max_links_per_ip", 20) self.set_var("connect_create_interval", 0) self.load() \ No newline at end of file diff --git a/code/default/x_tunnel/local/seley_front/connect_creator.py b/code/default/x_tunnel/local/seley_front/connect_creator.py index 595dc566a5..ba913ada32 100644 --- a/code/default/x_tunnel/local/seley_front/connect_creator.py +++ b/code/default/x_tunnel/local/seley_front/connect_creator.py @@ -55,7 +55,7 @@ def connect_ssl(self, ip_str, sni, host, close_cb=None): ip_str=ip_str, sni=sni, on_close=close_cb) - + ssl_sock.ip = ip ssl_sock.sni = utils.to_str(sni) time_connected = time.time() @@ -77,6 +77,9 @@ def connect_ssl(self, ip_str, sni, host, close_cb=None): ssl_sock.host = host ssl_sock.received_size = 0 + if self.debug: + self.logger.debug("connect ip:%s sni:%s host:%s success", ip_str, sni, host) + return ssl_sock def check_cert(self, ssl_sock): diff --git a/code/default/x_tunnel/local/seley_front/ip_manager.py b/code/default/x_tunnel/local/seley_front/ip_manager.py index dda2f8bcd1..716765dc49 100644 --- a/code/default/x_tunnel/local/seley_front/ip_manager.py +++ b/code/default/x_tunnel/local/seley_front/ip_manager.py @@ -2,6 +2,7 @@ import os import json import socket +import threading from front_base.ip_manager import IpManagerBase import utils @@ -16,6 +17,7 @@ def __init__(self, config, logger, config_fn, speed_fn): self.hosts = {} self.ip_dict = {} self.load() + threading.Thread(target=self.resolve_domains, name="seley_resolve_domain", daemon=False).start() def __str__(self): o = "" @@ -44,8 +46,11 @@ def load(self): xlog.warn("load hosts %s e:%r", self.config_fn, e) return + self.hosts = domain_hosts + + def resolve_domains(self): ip_hosts = {} - for domain_port, host_info in domain_hosts.items(): + for domain_port, host_info in self.hosts.items(): if not host_info.get("key"): continue @@ -81,8 +86,8 @@ def _get_ip_info(self, ip_str): def get_ip_sni_host(self): now = time.time() - ip_str = None best_info = None + best_params = {} best_speed = 0 for ip_str, params in self.hosts.items(): @@ -102,6 +107,7 @@ def get_ip_sni_host(self): if now - info["last_try"] > 30 * 60: best_info = info + best_params = params # xlog.debug("get_ip_sni_host last_try %s", ip_str) break @@ -109,17 +115,23 @@ def get_ip_sni_host(self): if speed > best_speed: best_speed = speed best_info = info + best_params = params # xlog.debug("get_ip_sni_host best speed %s", ip_str) if not best_info: - return None, None, None + return None ip_str = best_info["ip_str"] self.ip_dict[ip_str]["links"] += 1 self.ip_dict[ip_str]["last_try"] = now key = self.hosts[ip_str]["key"] - return best_info["ip_str"], key, ip_str + return { + "ip_str": best_info["ip_str"], + "sni": key, + "host": ip_str, + "adjust": best_params.get("adjust", 0), + } def update_ip(self, ip_str, sni, handshake_time): info = self._get_ip_info(ip_str) diff --git a/code/default/x_tunnel/local/tls_relay_front/ip_manager.py b/code/default/x_tunnel/local/tls_relay_front/ip_manager.py index ba14b73e11..368fd9c93c 100644 --- a/code/default/x_tunnel/local/tls_relay_front/ip_manager.py +++ b/code/default/x_tunnel/local/tls_relay_front/ip_manager.py @@ -28,13 +28,15 @@ def _get_ip_info(self, ip): def get_ip_sni_host(self): now = time.time() - ips = self.host_manager.ips best_info = None best_speed = 0 - for ip in ips: - port = int(self.host_manager.info[ip].get("port", 443)) + for ip, ip_info in self.host_manager.info.items(): + if "sni" not in ip_info: + continue + + port = int(ip_info.get("port", 443)) ip_str = utils.get_ip_str(ip, port) info = self._get_ip_info(ip) @@ -54,7 +56,7 @@ def get_ip_sni_host(self): best_info = info if not best_info: - return None, None, None + return None best_info["links"] += 1 best_info["last_try"] = now @@ -63,7 +65,11 @@ def get_ip_sni_host(self): ip = best_info["ip"] port = int(self.host_manager.info[ip].get("port", 443)) ip_str = utils.get_ip_str(ip, port) - return ip_str, None, None + return { + "ip_str": ip_str, + "sni": None, + "host": None, + } def update_ip(self, ip_str, sni, handshake_time): ip, _ = utils.get_ip_port(ip_str) diff --git a/code/default/x_tunnel/local/upload_logs.py b/code/default/x_tunnel/local/upload_logs.py index 31ef647b24..d389bc854e 100644 --- a/code/default/x_tunnel/local/upload_logs.py +++ b/code/default/x_tunnel/local/upload_logs.py @@ -43,7 +43,7 @@ def mask_x_tunnel_password(fp): def get_launcher_port(): launcher_config_fn = os.path.join(data_path, "launcher", "config.json") try: - with open(launcher_config_fn, "r") as fd: + with open(launcher_config_fn, "r", encoding='utf-8') as fd: info = json.load(fd) return info.get("control_port", 8085) except Exception as e: