feat: parallel limited time load of user META info (#71)

* feat: parallel limited time load of user META info Now Public IP, ASN, Region, ... are loaded using separated helper daemon process Data are shared using shared memory between two processes we have a minimum wait time to let helper process complete, if not ,main process will terminate assuming meta data cannot be found (No Internet or etc) * drop privileges * feat: serialize request Now request is made in serial but lack of timeout is handled using separated process * fix: fix privilege drop bypass for windows * cleanup * cleanup * cleanup * cleanup * cleanup * cleanup * Change from Process to Thread as some anti malware tools may get suspicious of Process * cleanup * [feat]: different method for windows and posix systems we use independent process in posix system to make privilege dropping stable and thread based approach in windows to prevent windows system get suspicious and block requests * [feat]: different method for windows and posix systems we use independent process in posix system to make privilege dropping stable and thread based approach in windows to prevent windows system get suspicious and block requests * [fix]: fix privilege dropping move unix privilege dropping into process itself * [fix] remove some duplicates * Update utils/geolocate.py Co-authored-by: Simone Basso <[email protected]> * Update utils/geolocate.py Co-authored-by: Simone Basso <[email protected]> * style 1/n * fix 1/n Co-authored-by: xhdix <[email protected]> Co-authored-by: Simone Basso <[email protected]>
wikicensorship · Sep 2, 2022 · 73363e2 · 73363e2
1 parent 133e83c
commit 73363e2
Show file tree

Hide file tree

Showing 2 changed files with 77 additions and 44 deletions.
diff --git a/utils/geolocate.py b/utils/geolocate.py
@@ -1,48 +1,23 @@
 #!/usr/bin/env python3
 
+import ctypes
 import json
 import os
 import platform
+import time
+from multiprocessing import Process, RawArray, Value
+from threading import Thread
 from urllib.request import Request, urlopen
 
-from scapy.all import DNS, DNSQR, DNSRR, IP, UDP, RandShort, sr
-
-import utils.ephemeral_port
-
 OS_NAME = platform.system()
 
 
-def nslookup(user_iface, user_source_ip_address):
-    # there is no timeout in getaddrinfo(), so we have to do it ourselves
-    # Raw packets bypasses the firewall so it may not work as intended in some cases
-    dns_request = IP(
-        src=user_source_ip_address, dst="1.1.1.1", id=RandShort(), ttl=128)/UDP(
-            sport=utils.ephemeral_port.ephemeral_port_reserve(
-                user_source_ip_address,  "udp"), dport=53)/DNS(
-                    rd=1, id=RandShort(), qd=DNSQR(qname="speed.cloudflare.com"))
-    try:
-        request_and_answers, _ = sr(
-            dns_request, iface=user_iface, verbose=0, timeout=1)
-    except:
-        return False
-    if request_and_answers is not None and len(request_and_answers) != 0:
-        if request_and_answers[0][1].haslayer(DNS):
-            return True
-            # return request_and_answers[0][1][DNSRR].rdata
-    return False
-
-
 def get_meta_json():
-    usereuid = None
     meta_url = 'https://speed.cloudflare.com/meta'
     # TODO(xhdix): change versioning
     httprequest = Request(
         meta_url, headers={'user-agent': 'TraceVis/0.7.0 (WikiCensorship)'})
     try:
-        if OS_NAME == "Linux":
-            if os.geteuid() == 0:
-                usereuid = os.geteuid()
-                os.seteuid(65534)  # user id of the user "nobody"
         with urlopen(httprequest, timeout=9) as response:
             if response.status == 200:
                 meta_json = json.load(response)
@@ -52,21 +27,24 @@ def get_meta_json():
     except Exception as e:
         print(f"Notice!\n{e!s}")
         return None
-    finally:
-        if usereuid != None:
-            os.seteuid(usereuid)
 
 
-def get_meta(user_iface, user_source_ip_address):
+def drop_privileges():
+    os.setgroups([])
+    os.setresgid(65534, 65534, 65534)
+    os.setresuid(65534, 65534, 65534)
+    os.umask(0o077)
+
+
+def get_meta_vars():
     no_internet = True
     public_ip = '127.1.2.7'  # we should know that what we are going to clean
     network_asn = 'AS0'
     network_name = ''
     country_code = ''
     city = ''
+
     print("· - · · · detecting IP, ASN, country, etc · - · · · ")
-    if not nslookup(user_iface, user_source_ip_address):
-        return no_internet, public_ip, network_asn, network_name, country_code, city
     user_meta = get_meta_json()
     if user_meta is not None:
         no_internet = False
@@ -75,7 +53,7 @@ def get_meta(user_iface, user_source_ip_address):
             print("· · · - · " + public_ip)
             print('. - . - . we use public IP to know what to remove from data!')
         if 'asn' in user_meta.keys():
-            network_asn = "AS" + str(user_meta['asn'])
+            network_asn = ("AS" + str(user_meta['asn']))
             print("· · · - · " + network_asn)
         if 'asOrganization' in user_meta.keys():
             network_name = user_meta['asOrganization']
@@ -87,3 +65,60 @@ def get_meta(user_iface, user_source_ip_address):
             city = user_meta['city']
             print("· · · - · " + city)
     return no_internet, public_ip, network_asn, network_name, country_code, city
+
+
+def posix_run_geolocate():
+    def get_meta(no_internet, public_ip, network_asn, network_name, country_code, city):
+        drop_privileges()
+        no_internet.value, public_ip.value, network_asn.value, network_name.value, country_code.value, city.value = get_meta_vars()
+
+    user_meta_info_timeout = 10   # Seconds
+    no_internet = Value(ctypes.c_bool, True)
+    public_ip = RawArray(ctypes.c_wchar, 40)
+    public_ip.value = '127.1.2.7'
+    network_asn = RawArray(ctypes.c_wchar, 100)
+    network_asn.value = 'AS0'
+    network_name = RawArray(ctypes.c_wchar, 100)
+    country_code = RawArray(ctypes.c_wchar, 100)
+    city = RawArray(ctypes.c_wchar, 100)
+    p = Process(target=get_meta, daemon=True, args=(
+        no_internet, public_ip, network_asn, network_name, country_code, city))
+    p.start()
+    user_meta_info_start_time = time.time()
+    while (time.time() - user_meta_info_start_time < user_meta_info_timeout) and no_internet.value:
+        time.sleep(1)
+
+    return no_internet.value, public_ip.value, network_asn.value, network_name.value, country_code.value, city.value
+
+
+def windows_run_geolocate():
+    def get_meta():
+        nonlocal no_internet, public_ip, network_asn, network_name, country_code, city
+        no_internet, public_ip, network_asn, network_name, country_code, city = get_meta_vars()
+
+    user_meta_info_timeout = 10   # Seconds
+    no_internet = True
+    public_ip = '127.1.2.7'  # we should know that what we are going to clean
+    network_asn = 'AS0'
+    network_name = ''
+    country_code = ''
+    city = ''
+    user_meta_info_start_time = 0
+    p = Thread(target=get_meta, daemon=True)
+    p.start()
+    user_meta_info_start_time = time.time()
+    while (time.time() - user_meta_info_start_time < user_meta_info_timeout) and no_internet:
+        time.sleep(1)
+
+    return no_internet, public_ip, network_asn, network_name, country_code, city
+
+
+def run_geolocate():
+    # threat windows and other posix systems differently
+    # windows get suspicious when we spawn an independent Process
+    # so we need to use thread for that
+    # in other posix systems we need dropping privilege and as
+    # this is not possible in python threads we stick to process for those systems
+    if os.name == "posix":
+        return posix_run_geolocate()
+    return windows_run_geolocate()
diff --git a/utils/trace.py b/utils/trace.py
@@ -529,14 +529,12 @@ def trace_route(
         paris_id = repeat_requests
     elif trace_retransmission:
         paris_id = -1
-    no_internet, public_ip, network_asn, network_name, country_code, city = utils.geolocate.get_meta(
-        user_iface, user_source_ip_address)
-    if name_prefix != "":
-        measurement_name = name_prefix + '-' + network_asn + "-tracevis-" + \
-            datetime.utcnow().strftime("%Y%m%d-%H%M")
-    else:
-        measurement_name = network_asn + "-tracevis-" + \
-            datetime.utcnow().strftime("%Y%m%d-%H%M")
+
+    no_internet, public_ip, network_asn, network_name, country_code, city = utils.geolocate.run_geolocate()
+
+    measurement_name = (f"{name_prefix}-{network_asn}-tracevis-" if name_prefix else f"{network_asn}-tracevis-") + \
+        datetime.utcnow().strftime("%Y%m%d-%H%M")
+
     initialize_json_first_nodes(
         request_ips=request_ips, annotation_1=annotation_1, annotation_2=annotation_2,
         packet_1_proto=p1_proto, packet_2_proto=p2_proto,