1.1.0 fix bugs

Author: debian

goagent/3.1.35/local/cert_util.py

@@ -14,6 +14,21 @@
 import logging
 
 
+current_path = os.path.dirname(os.path.abspath(__file__))
+python_path = os.path.abspath( os.path.join(current_path, os.pardir, os.pardir, os.pardir, 'python27', '1.0'))
+data_path = os.path.abspath(os.path.join(current_path, os.pardir, os.pardir, os.pardir, 'data', 'goagent'))
+if not os.path.isdir(data_path):
+    data_path = current_path
+
+noarch_lib = os.path.abspath( os.path.join(python_path, 'lib', 'noarch'))
+sys.path.append(noarch_lib)
+
+if sys.platform == "win32":
+    win32_lib = os.path.abspath( os.path.join(python_path, 'lib', 'win32'))
+    sys.path.append(win32_lib)
+elif sys.platform == "linux" or sys.platform == "linux2":
+    linux_lib = os.path.abspath( os.path.join(python_path, 'lib', 'linux'))
+    sys.path.append(linux_lib)
 
 import OpenSSL
 
@@ -123,9 +138,11 @@ class CertUtil(object):
     """CertUtil module, based on mitmproxy"""
 
     ca_vendor = 'GoAgent'
-    ca_keyfile = 'CA.crt'
+    ca_keyfile = os.path.join(data_path, 'CA.crt')
     ca_thumbprint = ''
-    ca_certdir = 'certs'
+    ca_certdir = os.path.join(data_path, 'certs')
+    if not os.path.isdir(ca_certdir):
+        os.mkdir(ca_certdir)
     ca_lock = threading.Lock()
 
     @staticmethod
@@ -247,7 +264,8 @@ def win32_notify( msg="msg", title="Title"):
 
     @staticmethod
     def import_ca(certfile):
-        commonname = os.path.splitext(os.path.basename(certfile))[0]
+        #commonname = os.path.splitext(os.path.basename(certfile))[0]
+        commonname = "GoAgent CA - GoAgent"
         if sys.platform.startswith('win'):
             import ctypes
             with open(certfile, 'rb') as fp:
@@ -298,8 +316,27 @@ class CRYPT_HASH_BLOB(ctypes.Structure):
                 if not os.path.exists(pemfile):
                     return os.system('cp "%s" "%s" && update-ca-certificates' % (certfile, new_certfile))
             elif any(os.path.isfile('%s/certutil' % x) for x in os.environ['PATH'].split(os.pathsep)):
-                cmd_line = 'certutil -L -d sql:$HOME/.pki/nssdb | grep "%s" || certutil -d sql:$HOME/.pki/nssdb -A -t "C,," -n "%s" -i "%s"' % (commonname, commonname, certfile)
-                return os.system(cmd_line)
+                # certutil -L -d sql:$HOME/.pki/nssdb | grep "%s" ||  % commonname,
+                # remove old cert first
+                cmd_line = 'certutil -d sql:$HOME/.pki/nssdb -D -n "%s" ' % commonname
+                os.system(cmd_line)
+
+                # install new cert
+                cmd_line = 'certutil -d sql:$HOME/.pki/nssdb -A -t "C,," -n "%s" -i "%s"' % (commonname, certfile)
+                os.system(cmd_line)
+
+                #Firefox:
+                firefox_config_path = get_linux_firefox_path()
+                if not firefox_config_path:
+                    return
+
+                cmd_line = 'certutil -d %s -D -n "%s" ' % (firefox_config_path, commonname)
+                os.system(cmd_line) # remove old cert first
+
+                cmd_line = 'certutil -d %s -A -t "C,," -n "%s" -i "%s"' % (firefox_config_path, commonname, certfile)
+                os.system(cmd_line) # install new cert
+
+                return 0
             else:
                 logging.warning('please install *libnss3-tools* package to import GoAgent root ca')
         return 0
@@ -362,7 +399,19 @@ def check_ca():
         if not os.path.exists(certdir):
             os.makedirs(certdir)
 
+def get_linux_firefox_path():
+    home_path = os.path.expanduser("~")
+    firefox_path = os.path.join(home_path, ".mozilla/firefox")
+    if not os.path.isdir(firefox_path):
+        return
+
+    for filename in os.listdir(firefox_path):
+        if filename.endswith(".default") and os.path.isdir(os.path.join(firefox_path, filename)):
+            config_path = os.path.join(firefox_path, filename)
+            return config_path
+
 
 if __name__ == '__main__':
     #capath = os.path.join(os.path.dirname(os.path.abspath(__file__)), CertUtil.ca_keyfile)
     CertUtil.check_ca()
+    #print get_linux_firefox_path()

goagent/3.1.35/local/connect_manager.py

@@ -1,12 +1,10 @@
 #!/usr/bin/env python
 # coding:utf-8
 
-import sys
 import os
 import errno
 import binascii
 import time
-import thread
 import socket
 import select
 import Queue
@@ -155,8 +153,8 @@ def __init__(self):
         self.max_retry = 3
         self.timeout = 3
         self.max_timeout = 5
-        self.max_thread_num = 40
-        self.connection_pool_num = 30
+        self.max_thread_num = 10
+        self.connection_pool_num = 20
 
         self.conn_pool = Connect_pool() #Queue.PriorityQueue()
 
@@ -173,10 +171,15 @@ def save_ssl_connection_for_reuse(self, ssl_sock):
 
         while self.conn_pool.qsize() > self.connection_pool_num:
             t, ssl_sock = self.conn_pool.get_slowest()
-            if t < 300:
+
+            if t < 500:
                 #self.conn_pool.put( (ssl_sock.handshake_time, ssl_sock) )
+                ssl_sock.close()
                 return
-            ssl_sock.close()
+            else:
+                ssl_sock.close()
+
+
 
     def create_ssl_connection(self):
 
@@ -268,6 +271,7 @@ def connect_thread():
                     if ssl_sock:
                         ssl_sock.last_use_time = time.time()
                         self.conn_pool.put((ssl_sock.handshake_time, ssl_sock))
+                    time.sleep(1)
             finally:
                 self.thread_num_lock.acquire()
                 self.thread_num -= 1
@@ -282,6 +286,7 @@ def create_more_connection():
                 p = threading.Thread(target = connect_thread)
                 p.daemon = True
                 p.start()
+                time.sleep(0.5)
 
 
         while True:
@@ -320,14 +325,19 @@ def create_more_connection():
 
 
 class Forward_connection_manager():
-    timeout = 3
-    max_timeout = 5
+    timeout = 1
+    max_timeout = 10
     tcp_connection_cache = Queue.PriorityQueue()
+    thread_num_lock = threading.Lock()
+    thread_num = 0
+    max_thread_num = 10
+
+    def create_connection(self, port=443, sock_life=5):
+        if port != 443:
+            logging.warn("forward port %d not supported.", port)
+            return None
 
-    def create_connection(self, sock_life=5):
-        def _create_connection(ip_port, queobj, delay=0):
-            if delay != 0:
-                time.sleep(delay)
+        def _create_connection(ip_port):
             ip = ip_port[0]
             sock = None
             # start connection time record
@@ -358,54 +368,52 @@ def _create_connection(ip_port, queobj, delay=0):
                 logging.debug("tcp conn %s time:%d", ip, conn_time * 1000)
 
                 # put ssl socket object to output queobj
-                queobj.put(sock)
+                self.tcp_connection_cache.put((time.time(), sock))
             except Exception as e:
-                # any socket.error, put Excpetions to output queobj.
-                queobj.put(e)
                 conn_time = int((time.time() - start_time) * 1000)
                 logging.debug("tcp conn %s fail t:%d", ip, conn_time)
                 google_ip.report_connect_fail(ip)
                 #logging.info("create_tcp report fail ip:%s", ip)
                 if sock:
                     sock.close()
+            finally:
+                self.thread_num_lock.acquire()
+                self.thread_num -= 1
+                self.thread_num_lock.release()
 
-        def recycle_connection(count, queobj):
-            for i in range(count):
-                sock = queobj.get()
-                if sock and not isinstance(sock, Exception):
-                    self.tcp_connection_cache.put((time.time(), sock))
-
-        try:
-            ctime, sock = self.tcp_connection_cache.get_nowait()
-            if time.time() - ctime < sock_life:
-                return sock
-        except Queue.Empty:
-            pass
 
+        while True:
+            try:
+                ctime, sock = self.tcp_connection_cache.get_nowait()
+                if time.time() - ctime < sock_life:
+                    return sock
+                else:
+                    sock.close()
+                    continue
+            except Queue.Empty:
+                break
 
-        port = 443
         start_time = time.time()
-        #while time.time() - start_time < self.max_timeout:
-        for j in range(3):
-            addresses = []
-            for i in range(3):
+        while time.time() - start_time < self.max_timeout:
+
+            if self.thread_num < self.max_thread_num:
                 ip = google_ip.get_gws_ip()
                 if not ip:
-                    logging.warning("no gws ip.")
+                    logging.error("no gws ip.")
                     return
-                addresses.append((ip, port))
-
-            addrs = addresses
-            queobj = Queue.Queue()
-            delay = 0
-            for addr in addrs:
-                thread.start_new_thread(_create_connection, (addr, queobj, delay))
-                #delay += 0.05
-            for i in range(len(addrs)):
-                result = queobj.get()
-                if not isinstance(result, (socket.error, OSError, IOError)):
-                    thread.start_new_thread(recycle_connection, (len(addrs)-i-1, queobj))
-                    return result
+                addr = (ip, port)
+                self.thread_num_lock.acquire()
+                self.thread_num += 1
+                self.thread_num_lock.release()
+                p = threading.Thread(target=_create_connection, args=(addr,))
+                p.daemon = True
+                p.start()
+
+            try:
+                ctime, sock = self.tcp_connection_cache.get(timeout=0.4)
+                return sock
+            except:
+                continue
         logging.warning('create tcp connection fail.')
 
 
@@ -425,6 +433,10 @@ def forward_socket(self, local, remote, timeout=60, tick=2, bufsize=8192):
                 for sock in ins:
                     data = sock.recv(bufsize)
                     if not data:
+                        if sock is remote:
+                            logging.debug("forward remote disconnected.")
+                        else:
+                            logging.debug("forward local disconnected.")
                         return
 
                     if sock is remote:
@@ -433,15 +445,15 @@ def forward_socket(self, local, remote, timeout=60, tick=2, bufsize=8192):
                     else:
                         remote.sendall(data)
                         timecount = timeout
-        except NetWorkIOError as e:
+        except Exception as e:
             if e.args[0] not in (errno.ECONNABORTED, errno.ECONNRESET, errno.ENOTCONN, errno.EPIPE):
-                raise
+                logging.exception("forward except:%s.", e)
         finally:
             if local:
                 local.close()
             if remote:
                 remote.close()
-            logging.debug("forward closed.")
+
 
 
 

goagent/3.1.35/local/google_ip.py

@@ -23,7 +23,7 @@
 
 
 # const value:
-max_check_ip_thread_num = 10
+max_check_ip_thread_num = 5
 max_good_ip_num = 4000  # stop scan ip when enough
 
 
@@ -46,6 +46,7 @@ class Check_ip():
     gws_ip_list = [] # gererate from ip_dict, sort by handshake_time, when get_batch_ip
     ip_lock = threading.Lock()
     iplist_need_save = 0
+    iplist_saved_time = 0
     last_sort_time_for_gws = 0  # keep status for avoid wast too many cpu
 
     network_fail_time = 0 # keep status for avoid retry too frequently
@@ -103,8 +104,13 @@ def load_ip(self):
             p.start()
 
     def save_ip_list(self, force=False):
-        if self.iplist_need_save == 0 and not force:
-            return
+        if not force:
+            if self.iplist_need_save == 0:
+                return
+            if time.time() - self.iplist_saved_time < 10:
+                return
+
+        self.iplist_saved_time = time.time()
 
         try:
             self.ip_lock.acquire()
@@ -408,6 +414,8 @@ def check_exist_ip(self):
                 self.update_ip(ip_str, result.handshake_time)
                 logging.info("check_exist_ip update ip:%s server:%s time:%d", ip_str, result.server_type, result.handshake_time)
 
+            time.sleep(1)
+
         self.save_ip_list()
 
 if __name__ != "__main__":

goagent/3.1.35/local/pac_server.py

@@ -4,20 +4,6 @@
 import sys
 import os
 import glob
-
-sys.path += glob.glob('%s/*.egg' % os.path.dirname(os.path.abspath(__file__)))
-
-try:
-    import gevent
-    import gevent.socket
-    import gevent.server
-    import gevent.queue
-    import gevent.event
-    import gevent.monkey
-    gevent.monkey.patch_all(subprocess=True)
-except ImportError:
-    gevent = None
-
 import base64
 import time
 import re
@@ -26,14 +12,6 @@
 import BaseHTTPServer
 import urllib2
 import urlparse
-try:
-    import OpenSSL
-except ImportError:
-    OpenSSL = None
-try:
-    import dnslib
-except ImportError:
-    dnslib = None
 
 
 from config import config
@@ -69,10 +47,7 @@ def update_pacfile(filename):
                 logging.info('try download %r to update_pacfile(%r)', config.PAC_ADBLOCK, filename)
                 adblock_content = opener.open(config.PAC_ADBLOCK).read()
                 logging.info('%r downloaded, try convert it with adblock2pac', config.PAC_ADBLOCK)
-                if 'gevent' in sys.modules and time.sleep is getattr(sys.modules['gevent'], 'sleep', None) and hasattr(gevent.get_hub(), 'threadpool'):
-                    jsrule = gevent.get_hub().threadpool.apply_e(Exception, PacUtil.adblock2pac, (adblock_content, 'FindProxyForURLByAdblock', blackhole, default))
-                else:
-                    jsrule = PacUtil.adblock2pac(adblock_content, 'FindProxyForURLByAdblock', blackhole, default)
+                jsrule = PacUtil.adblock2pac(adblock_content, 'FindProxyForURLByAdblock', blackhole, default)
                 content += '\r\n' + jsrule + '\r\n'
                 logging.info('%r downloaded and parsed', config.PAC_ADBLOCK)
             else:
@@ -87,10 +62,7 @@ def update_pacfile(filename):
             logging.info('try download %r to update_pacfile(%r)', config.PAC_GFWLIST, filename)
             autoproxy_content = base64.b64decode(opener.open(config.PAC_GFWLIST).read())
             logging.info('%r downloaded, try convert it with autoproxy2pac', config.PAC_GFWLIST)
-            if 'gevent' in sys.modules and time.sleep is getattr(sys.modules['gevent'], 'sleep', None) and hasattr(gevent.get_hub(), 'threadpool'):
-                jsrule = gevent.get_hub().threadpool.apply_e(Exception, PacUtil.autoproxy2pac, (autoproxy_content, 'FindProxyForURLByAutoProxy', autoproxy, default))
-            else:
-                jsrule = PacUtil.autoproxy2pac(autoproxy_content, 'FindProxyForURLByAutoProxy', autoproxy, default)
+            jsrule = PacUtil.autoproxy2pac(autoproxy_content, 'FindProxyForURLByAutoProxy', autoproxy, default)
             content += '\r\n' + jsrule + '\r\n'
             logging.info('%r downloaded and parsed', config.PAC_GFWLIST)
         except Exception as e: