From 73363e21ad82d2ce7a12cf81e1dbb58f44c074eb Mon Sep 17 00:00:00 2001 From: Ariyan Eghbal Date: Fri, 2 Sep 2022 22:32:10 +0430 Subject: [PATCH] feat: parallel limited time load of user META info (#71) * feat: parallel limited time load of user META info Now Public IP, ASN, Region, ... are loaded using separated helper daemon process Data are shared using shared memory between two processes we have a minimum wait time to let helper process complete, if not ,main process will terminate assuming meta data cannot be found (No Internet or etc) * drop privileges * feat: serialize request Now request is made in serial but lack of timeout is handled using separated process * fix: fix privilege drop bypass for windows * cleanup * cleanup * cleanup * cleanup * cleanup * cleanup * Change from Process to Thread as some anti malware tools may get suspicious of Process * cleanup * [feat]: different method for windows and posix systems we use independent process in posix system to make privilege dropping stable and thread based approach in windows to prevent windows system get suspicious and block requests * [feat]: different method for windows and posix systems we use independent process in posix system to make privilege dropping stable and thread based approach in windows to prevent windows system get suspicious and block requests * [fix]: fix privilege dropping move unix privilege dropping into process itself * [fix] remove some duplicates * Update utils/geolocate.py Co-authored-by: Simone Basso * Update utils/geolocate.py Co-authored-by: Simone Basso * style 1/n * fix 1/n Co-authored-by: xhdix Co-authored-by: Simone Basso --- utils/geolocate.py | 107 ++++++++++++++++++++++++++++++--------------- utils/trace.py | 14 +++--- 2 files changed, 77 insertions(+), 44 deletions(-) diff --git a/utils/geolocate.py b/utils/geolocate.py index c3265f1..c795d7a 100644 --- a/utils/geolocate.py +++ b/utils/geolocate.py @@ -1,48 +1,23 @@ #!/usr/bin/env python3 +import ctypes import json import os import platform +import time +from multiprocessing import Process, RawArray, Value +from threading import Thread from urllib.request import Request, urlopen -from scapy.all import DNS, DNSQR, DNSRR, IP, UDP, RandShort, sr - -import utils.ephemeral_port - OS_NAME = platform.system() -def nslookup(user_iface, user_source_ip_address): - # there is no timeout in getaddrinfo(), so we have to do it ourselves - # Raw packets bypasses the firewall so it may not work as intended in some cases - dns_request = IP( - src=user_source_ip_address, dst="1.1.1.1", id=RandShort(), ttl=128)/UDP( - sport=utils.ephemeral_port.ephemeral_port_reserve( - user_source_ip_address, "udp"), dport=53)/DNS( - rd=1, id=RandShort(), qd=DNSQR(qname="speed.cloudflare.com")) - try: - request_and_answers, _ = sr( - dns_request, iface=user_iface, verbose=0, timeout=1) - except: - return False - if request_and_answers is not None and len(request_and_answers) != 0: - if request_and_answers[0][1].haslayer(DNS): - return True - # return request_and_answers[0][1][DNSRR].rdata - return False - - def get_meta_json(): - usereuid = None meta_url = 'https://speed.cloudflare.com/meta' # TODO(xhdix): change versioning httprequest = Request( meta_url, headers={'user-agent': 'TraceVis/0.7.0 (WikiCensorship)'}) try: - if OS_NAME == "Linux": - if os.geteuid() == 0: - usereuid = os.geteuid() - os.seteuid(65534) # user id of the user "nobody" with urlopen(httprequest, timeout=9) as response: if response.status == 200: meta_json = json.load(response) @@ -52,21 +27,24 @@ def get_meta_json(): except Exception as e: print(f"Notice!\n{e!s}") return None - finally: - if usereuid != None: - os.seteuid(usereuid) -def get_meta(user_iface, user_source_ip_address): +def drop_privileges(): + os.setgroups([]) + os.setresgid(65534, 65534, 65534) + os.setresuid(65534, 65534, 65534) + os.umask(0o077) + + +def get_meta_vars(): no_internet = True public_ip = '127.1.2.7' # we should know that what we are going to clean network_asn = 'AS0' network_name = '' country_code = '' city = '' + print("· - · · · detecting IP, ASN, country, etc · - · · · ") - if not nslookup(user_iface, user_source_ip_address): - return no_internet, public_ip, network_asn, network_name, country_code, city user_meta = get_meta_json() if user_meta is not None: no_internet = False @@ -75,7 +53,7 @@ def get_meta(user_iface, user_source_ip_address): print("· · · - · " + public_ip) print('. - . - . we use public IP to know what to remove from data!') if 'asn' in user_meta.keys(): - network_asn = "AS" + str(user_meta['asn']) + network_asn = ("AS" + str(user_meta['asn'])) print("· · · - · " + network_asn) if 'asOrganization' in user_meta.keys(): network_name = user_meta['asOrganization'] @@ -87,3 +65,60 @@ def get_meta(user_iface, user_source_ip_address): city = user_meta['city'] print("· · · - · " + city) return no_internet, public_ip, network_asn, network_name, country_code, city + + +def posix_run_geolocate(): + def get_meta(no_internet, public_ip, network_asn, network_name, country_code, city): + drop_privileges() + no_internet.value, public_ip.value, network_asn.value, network_name.value, country_code.value, city.value = get_meta_vars() + + user_meta_info_timeout = 10 # Seconds + no_internet = Value(ctypes.c_bool, True) + public_ip = RawArray(ctypes.c_wchar, 40) + public_ip.value = '127.1.2.7' + network_asn = RawArray(ctypes.c_wchar, 100) + network_asn.value = 'AS0' + network_name = RawArray(ctypes.c_wchar, 100) + country_code = RawArray(ctypes.c_wchar, 100) + city = RawArray(ctypes.c_wchar, 100) + p = Process(target=get_meta, daemon=True, args=( + no_internet, public_ip, network_asn, network_name, country_code, city)) + p.start() + user_meta_info_start_time = time.time() + while (time.time() - user_meta_info_start_time < user_meta_info_timeout) and no_internet.value: + time.sleep(1) + + return no_internet.value, public_ip.value, network_asn.value, network_name.value, country_code.value, city.value + + +def windows_run_geolocate(): + def get_meta(): + nonlocal no_internet, public_ip, network_asn, network_name, country_code, city + no_internet, public_ip, network_asn, network_name, country_code, city = get_meta_vars() + + user_meta_info_timeout = 10 # Seconds + no_internet = True + public_ip = '127.1.2.7' # we should know that what we are going to clean + network_asn = 'AS0' + network_name = '' + country_code = '' + city = '' + user_meta_info_start_time = 0 + p = Thread(target=get_meta, daemon=True) + p.start() + user_meta_info_start_time = time.time() + while (time.time() - user_meta_info_start_time < user_meta_info_timeout) and no_internet: + time.sleep(1) + + return no_internet, public_ip, network_asn, network_name, country_code, city + + +def run_geolocate(): + # threat windows and other posix systems differently + # windows get suspicious when we spawn an independent Process + # so we need to use thread for that + # in other posix systems we need dropping privilege and as + # this is not possible in python threads we stick to process for those systems + if os.name == "posix": + return posix_run_geolocate() + return windows_run_geolocate() diff --git a/utils/trace.py b/utils/trace.py index 233b04d..1e10b8a 100755 --- a/utils/trace.py +++ b/utils/trace.py @@ -529,14 +529,12 @@ def trace_route( paris_id = repeat_requests elif trace_retransmission: paris_id = -1 - no_internet, public_ip, network_asn, network_name, country_code, city = utils.geolocate.get_meta( - user_iface, user_source_ip_address) - if name_prefix != "": - measurement_name = name_prefix + '-' + network_asn + "-tracevis-" + \ - datetime.utcnow().strftime("%Y%m%d-%H%M") - else: - measurement_name = network_asn + "-tracevis-" + \ - datetime.utcnow().strftime("%Y%m%d-%H%M") + + no_internet, public_ip, network_asn, network_name, country_code, city = utils.geolocate.run_geolocate() + + measurement_name = (f"{name_prefix}-{network_asn}-tracevis-" if name_prefix else f"{network_asn}-tracevis-") + \ + datetime.utcnow().strftime("%Y%m%d-%H%M") + initialize_json_first_nodes( request_ips=request_ips, annotation_1=annotation_1, annotation_2=annotation_2, packet_1_proto=p1_proto, packet_2_proto=p2_proto,