From 9f43428cc9f8cd7f2ac1b050834072ef76a55962 Mon Sep 17 00:00:00 2001 From: "Michael.X" Date: Fri, 30 Jan 2015 23:57:39 -0400 Subject: [PATCH] v1.0.4 --- data/goagent/config.yaml | 8 +- goagent/3.1.32/local/cert_util.py | 2 +- goagent/3.1.32/local/check_ip.py | 2 +- goagent/3.1.32/local/connect_manager.py | 159 +++++++++-- goagent/3.1.32/local/good_ip.txt | 195 ++++++------- goagent/3.1.32/local/google_ip.py | 261 ++++++++++++------ goagent/3.1.32/local/proxy.ini | 8 +- goagent/3.1.32/local/proxy_handler.py | 28 +- goagent/3.1.32/local/remote_control.py | 21 ++ launcher/1.0.0/config.py | 41 ++- launcher/1.0.0/html/block/config.html | 54 +++- launcher/1.0.0/html/block/goagent/config.html | 2 + launcher/1.0.0/html/block/goagent/deploy.html | 2 +- .../1.0.0/html/block/goagent/logging.html | 2 +- launcher/1.0.0/html/block/goagent/status.html | 4 + launcher/1.0.0/start.py | 6 +- launcher/1.0.0/web_control.py | 14 +- 17 files changed, 567 insertions(+), 242 deletions(-) diff --git a/data/goagent/config.yaml b/data/goagent/config.yaml index db540e2119..0641b80cf0 100644 --- a/data/goagent/config.yaml +++ b/data/goagent/config.yaml @@ -1,5 +1,3 @@ -modules: - goagent: {auto_start: 1, current_version: 3.1.31, ignore_version: 3.1.31} - launcher: {current_version: 1.0.0, ignore_version: 1.0.0} -update: {check_update: 0, last_path: /media/release/XX-Net/launcher/1.0.0, node_id: !!python/long '8796754053427', - uuid: 63b44f9f-8256-4f44-a512-1da2bdebb62b} +socks_proxy: + {host:"", port:1080, user:"", password:""} +appids: {appids: "", password:""} diff --git a/goagent/3.1.32/local/cert_util.py b/goagent/3.1.32/local/cert_util.py index 4abd721936..23b08eafa2 100644 --- a/goagent/3.1.32/local/cert_util.py +++ b/goagent/3.1.32/local/cert_util.py @@ -375,5 +375,5 @@ def check_ca(): if __name__ == '__main__': - capath = os.path.join(os.path.dirname(os.path.abspath(__file__)), CertUtil.ca_keyfile) + #capath = os.path.join(os.path.dirname(os.path.abspath(__file__)), CertUtil.ca_keyfile) CertUtil.check_ca() diff --git a/goagent/3.1.32/local/check_ip.py b/goagent/3.1.32/local/check_ip.py index fe97187b82..2b08475b42 100644 --- a/goagent/3.1.32/local/check_ip.py +++ b/goagent/3.1.32/local/check_ip.py @@ -236,7 +236,7 @@ def test_main(): if __name__ == "__main__": #test("203.165.14.230", 10) #gws - test('208.117.224.213', 10) + test('210.139.253.39', 100) #test("218.176.242.24") #test_main() diff --git a/goagent/3.1.32/local/connect_manager.py b/goagent/3.1.32/local/connect_manager.py index 3ada7ed3ae..d0dbac6bc4 100644 --- a/goagent/3.1.32/local/connect_manager.py +++ b/goagent/3.1.32/local/connect_manager.py @@ -19,6 +19,7 @@ import logging from config import config import threading +import operator current_path = os.path.dirname(os.path.abspath(__file__)) python_path = os.path.abspath( os.path.join(current_path, os.pardir, os.pardir, os.pardir, 'python27', '1.0')) @@ -41,6 +42,60 @@ g_cacertfile = os.path.join(current_path, "cacert.pem") +class Connect_pool(object): + pool_lock = threading.Lock() + not_empty = threading.Condition(pool_lock) + pool = {} + + def qsize(self): + return len(self.pool) + + def put(self, item): + speed, sock = item + self.pool_lock.acquire() + try: + self.pool[sock] = speed + self.not_empty.notify() + finally: + self.pool_lock.release() + + def get(self, block=True): + self.not_empty.acquire() + try: + if not block: + if not self.qsize(): + raise + else: + while not self.qsize(): + self.not_empty.wait() + item = self._get() + return item + finally: + self.not_empty.release() + + def get_nowait(self): + return self.get(False) + + def _get(self): + #pool = sorted(self.pool.items(), key=operator.itemgetter(1)) + #k,v = pool[0] + #self.pool.pop(k) + #return (v, k) + + fastest_time = 9999 + fastest_sock = None + for sock in self.pool: + time = self.pool[sock] + if time < fastest_time: + fastest_time = time + fastest_sock = sock + + self.pool.pop(fastest_sock) + return (fastest_time, fastest_sock) + + + + class Https_connection_manager(object): thread_num_lock = threading.Lock() @@ -102,10 +157,10 @@ def __init__(self): self.max_retry = 3 self.timeout = 3 self.max_timeout = 5 - self.max_thread_num = 30 - self.min_connection_num = 30 + self.max_thread_num = 20 + self.min_connection_num = 20 - self.conn_pool = Queue.Queue() + self.conn_pool = Connect_pool() #Queue.PriorityQueue() # set_ciphers as Modern Browsers @@ -119,8 +174,15 @@ def __init__(self): if hasattr(OpenSSL.SSL, 'SESS_CACHE_BOTH'): self.openssl_context.set_session_cache_mode(OpenSSL.SSL.SESS_CACHE_BOTH) - def save_ssl_connection_for_reuse(self, socket): - self.conn_pool.put( (time.time(), socket) ) + def save_ssl_connection_for_reuse(self, ssl_sock): + if self.conn_pool.qsize() > 5: + if ssl_sock.handshake_time > 200 and ssl_sock.create_time - time.time() > 10: + return + if self.conn_pool.qsize() > self.min_connection_num - 5: + if ssl_sock.handshake_time > 300: + return + ssl_sock.last_use_time = time.time() + self.conn_pool.put( (ssl_sock.handshake_time, ssl_sock) ) def create_ssl_connection(self): @@ -160,9 +222,12 @@ def _create_ssl_connection(ip_port): handshake_time = int((time_handshaked - time_connected) * 1000) google_ip.update_ip(ip, handshake_time) - + #logging.debug("create_ssl update ip:%s time:%d", ip, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. + ssl_sock.ip = ip ssl_sock.sock = sock + ssl_sock.create_time = time_begin + ssl_sock.handshake_time = handshake_time # verify SSL certificate issuer. def check_ssl_cert(ssl_sock): @@ -178,9 +243,10 @@ def check_ssl_cert(ssl_sock): return ssl_sock except Exception as e: - logging.debug("create_ssl %s fail:%s", ip, e) + #logging.debug("create_ssl %s fail:%s", ip, e) google_ip.report_connect_fail(ip) + if ssl_sock: ssl_sock.close() if sock: @@ -197,10 +263,11 @@ def connect_thread(): break port = 443 - logging.debug("create ssl conn %s", ip_str) + #logging.debug("create ssl conn %s", ip_str) ssl_sock = _create_ssl_connection( (ip_str, port) ) if ssl_sock: - self.conn_pool.put((time.time(), ssl_sock)) + ssl_sock.last_use_time = time.time() + self.conn_pool.put((ssl_sock.handshake_time, ssl_sock)) finally: self.thread_num_lock.acquire() self.thread_num -= 1 @@ -219,30 +286,31 @@ def create_more_connection(): while True: try: - ctime, sock = self.conn_pool.get_nowait() + handshake_time, ssl_sock = self.conn_pool.get_nowait() except: - sock = None + ssl_sock = None break - if time.time() - ctime < 210: # gws ssl connection can keep for 230s after created + if time.time() - ssl_sock.last_use_time < 210: # gws ssl connection can keep for 230s after created + #logging.debug("ssl_pool.get:%s handshake:%d", ssl_sock.ip, handshake_time) break else: - sock.close() + ssl_sock.close() continue conn_num = self.conn_pool.qsize() - logging.debug("ssl conn_num:%d", conn_num) + #logging.debug("ssl conn_num:%d", conn_num) if conn_num < self.min_connection_num: create_more_connection() - if sock: - return sock + if ssl_sock: + return ssl_sock else: try: - ctime, sock = self.conn_pool.get() - return sock + handshake_time, ssl_sock = self.conn_pool.get() + return ssl_sock except Exception as e: - logging.warning("get ssl_pool err:%s", e) + logging.error("get ssl_pool err:%s", e) return None @@ -317,7 +385,9 @@ class Forward_connection_manager(): def create_connection(self, sock_life=5, cache_key=None): connection_cache_key = cache_key - def _create_connection(ip_port, timeout, queobj): + def _create_connection(ip_port, timeout, queobj, delay=0): + if delay != 0: + time.sleep(delay) ip = ip_port[0] sock = None try: @@ -339,14 +409,17 @@ def _create_connection(ip_port, timeout, queobj): # record TCP connection time conn_time = time.time() - start_time google_ip.update_ip(ip, conn_time * 2000) + logging.info("create_tcp update ip:%s time:%d", ip, conn_time * 2000) + logging.debug("tcp conn %s time:%d", ip, conn_time * 1000) # put ssl socket object to output queobj queobj.put(sock) except (socket.error, OSError) as e: # any socket.error, put Excpetions to output queobj. queobj.put(e) - + logging.debug("tcp conn %s fail", ip) google_ip.report_connect_fail(ip) + logging.info("create_ssl report fail ip:%s", ip) if sock: sock.close() @@ -362,7 +435,7 @@ def recycle_connection(count, queobj): if connection_cache_key: try: ctime, sock = self.tcp_connection_cache[connection_cache_key].get_nowait() - if time.time() - ctime < 5: + if time.time() - ctime < sock_life: return sock except Queue.Empty: pass @@ -376,17 +449,16 @@ def recycle_connection(count, queobj): addrs = addresses queobj = Queue.Queue() + delay = 0 for addr in addrs: - thread.start_new_thread(_create_connection, (addr, timeout, queobj)) + thread.start_new_thread(_create_connection, (addr, timeout, queobj, delay)) + delay += 0.01 for i in range(len(addrs)): result = queobj.get() if not isinstance(result, (socket.error, OSError)): thread.start_new_thread(recycle_connection, (len(addrs)-i-1, queobj)) return result - else: - if i == 0: - # only output first error - logging.warning('create_connection to %s return %r, try again.', addrs, result) + logging.warning('create_connection to %s fail.', addrs) def forward_socket(self, local, remote, timeout=60, tick=2, bufsize=8192): @@ -424,5 +496,38 @@ def forward_socket(self, local, remote, timeout=60, tick=2, bufsize=8192): logging.debug("forward closed.") + https_manager = Https_connection_manager() forwork_manager = Forward_connection_manager() + + +def test_pool(): + pool = Connect_pool() + pool.put((3, "c")) + pool.put((1, "a")) + pool.put((2, "b")) + + t, s = pool.get() + print s + + t, s = pool.get() + print s + + t, s = pool.get() + print s + + +def test_pool_speed(): + pool = Connect_pool() + for i in range(100): + pool.put((i, "%d"%i)) + + start = time.time() + t, s = pool.get() + print time.time() - start + print s + # sort time is 5ms for 10000 + # sort time is 0ms for 100 + +if __name__ == "__main__": + test_pool_speed() \ No newline at end of file diff --git a/goagent/3.1.32/local/good_ip.txt b/goagent/3.1.32/local/good_ip.txt index 9b301ca80b..b336d7a717 100644 --- a/goagent/3.1.32/local/good_ip.txt +++ b/goagent/3.1.32/local/good_ip.txt @@ -1,95 +1,100 @@ -106.162.216.89 google.com gws 1573 -64.15.112.151 google.com gws 978 -64.15.112.150 google.com gws 950 -203.117.34.177 google.com gws 3956 -106.162.216.20 google.com gws 4962 -64.15.126.134 www.google.com gws 1429 -64.233.160.142 *.au.doubleclick.net gws 632 -208.117.229.186 google.com gws 1053 -64.233.171.197 *.googleusercontent.com gws 866 -84.235.77.178 google.com gws 1513 -203.66.124.222 google.com gws 1303 -64.233.163.124 *.gfsvc.com gws 776 -203.66.124.182 google.com gws 2362 -74.125.195.165 *.googleadservices.com gws 841 -210.153.73.99 google.com gws 1213 -64.233.185.165 *.googleadservices.com gws 1719 -208.117.227.24 google.com gws 1004 -64.233.165.19 mail.google.com gws 891 -210.61.221.173 google.com gws 945 -64.233.180.137 *.googlegroups.com gws 947 -208.117.228.18 google.com gws 968 -74.125.129.165 *.googleadservices.com gws 789 -64.15.114.23 google.com gws 1476 -64.15.114.26 google.com gws 753 -213.240.44.23 google.com gws 3946 -208.117.228.57 google.com gws 815 -64.15.120.24 google.com gws 1026 -123.205.251.88 google.com gws 3671 -64.233.186.60 *.google.ch gws 1091 -64.15.119.230 www.google.com gws 1522 -64.233.178.117 upload.video.google.com gws 789 -64.233.179.31 *.google.com gws 804 -85.182.250.241 google.com gws 579 -64.233.186.155 *.g.doubleclick.net gws 1029 -106.162.216.59 google.com gws 256 -64.15.126.203 www.google.com gws 680 -64.233.178.133 *.googleusercontent.com gws 641 -64.15.126.190 www.google.com gws 1083 -64.15.126.194 www.google.com gws 1886 -64.233.179.102 *.google.com gws 692 -64.233.177.176 *.vp.video.l.google.com gws 618 -208.117.228.219 google.com gws 784 -64.233.171.199 google.com gws 1012 -64.233.177.95 *.googleapis.com gws 847 -173.194.71.144 *.uk.doubleclick.net gws 749 -64.233.182.147 www.google.com gws 1354 -64.233.171.100 *.google.com gws 816 -208.117.226.25 google.com gws 1504 -212.188.15.157 google.com gws 580 -212.188.15.153 google.com gws 627 -64.233.179.18 mail.google.com gws 679 -64.233.162.104 www.google.com gws 2103 -106.162.198.93 google.com gws 969 -111.168.255.173 google.com gws 4402 -123.205.250.112 google.com gws 4238 -210.139.253.173 google.com gws 4569 -64.15.126.146 www.google.com gws 669 -74.125.26.96 www.googleadservices.com gws 780 -123.205.251.117 google.com gws 1019 -208.117.229.69 www.google.com gws 1853 -64.233.171.112 adwords.google.com gws 804 -118.174.27.123 google.com gws 848 -203.165.13.223 google.com gws 4636 -74.125.228.220 *.doubleclick.net gws 2963 -212.188.15.148 google.com gws 896 -64.233.177.199 google.com gws 1842 -203.165.14.238 google.com gws 4261 -64.233.161.115 checkout.google.com gws 838 -64.15.126.254 www.google.com gws 1086 -208.117.228.48 google.com gws 701 -208.117.224.228 www.google.com gws 673 -64.15.126.152 www.google.com gws 2831 -60.199.175.88 google.com gws 3463 -64.15.126.159 www.google.com gws 693 -64.15.124.185 google.com gws 1286 -178.45.251.94 google.com gws 1242 -64.233.179.87 *.orkut.com gws 742 -64.15.117.120 google.com gws 1664 -210.242.125.25 google.com gws 4916 -208.117.230.84 google.com gws 1031 -208.117.224.195 www.google.com gws 2231 -64.15.114.52 google.com gws 1204 -212.188.15.177 google.com gws 550 -202.39.143.109 google.com gws 4656 -64.233.186.176 *.vp.video.l.google.com gws 894 -195.249.20.231 google.com gws 4883 -64.233.162.93 *.google.com gws 644 -203.117.34.163 google.com gws 4242 -64.15.126.242 www.google.com gws 605 -106.162.216.98 google.com gws 4170 -210.153.73.25 google.com gws 4639 -178.45.251.114 google.com gws 4807 -203.165.14.210 google.com gws 1355 -64.233.162.193 m.google.com gws 2795 -123.205.250.181 google.com gws 3361 +74.125.192.81 sandbox.google.com gws 649 +208.117.227.57 google.com gws 448 +64.233.181.195 google.com gws 572 +208.117.228.57 google.com gws 764 +106.162.192.172 google.com gws 179 +64.233.185.176 *.vp.video.l.google.com gws 609 +208.117.227.59 google.com gws 610 +64.233.167.103 www.google.com gws 700 +210.61.221.163 google.com gws 501 +64.233.163.124 *.gfsvc.com gws 644 +74.125.30.112 adwords.google.com gws 595 +208.117.229.91 www.google.com gws 957 +208.117.229.92 www.google.com gws 2159 +64.233.182.105 www.google.com gws 572 +85.182.250.227 google.com gws 3131 +208.117.233.23 google.com gws 547 +64.15.112.148 google.com gws 859 +106.162.198.103 google.com gws 361 +111.168.255.84 google.com gws 663 +84.235.77.49 google.com gws 1123 +64.233.186.79 *.google.dk gws 805 +64.233.186.78 *.google.es gws 981 +208.117.253.117 google.com gws 468 +64.233.177.124 *.gfsvc.com gws 618 +64.15.119.209 www.google.com gws 618 +64.15.119.207 www.google.com gws 549 +64.15.119.200 www.google.com gws 518 +81.175.29.185 google.com gws 621 +208.117.227.27 google.com gws 776 +208.117.227.24 google.com gws 687 +86.127.118.168 google.com gws 674 +64.233.165.19 mail.google.com gws 606 +64.233.189.117 upload.video.google.com gws 173 +86.127.118.162 google.com gws 685 +64.233.165.17 mail.google.com gws 617 +64.233.180.137 *.googlegroups.com gws 561 +64.15.124.244 google.com gws 613 +64.15.124.245 google.com gws 481 +202.39.143.54 google.com gws 90 +64.233.163.194 google.com gws 710 +213.240.44.25 google.com gws 826 +64.15.114.23 google.com gws 587 +64.15.114.26 google.com gws 512 +64.15.114.27 google.com gws 451 +64.15.114.24 google.com gws 440 +208.117.233.55 google.com gws 419 +64.15.120.24 google.com gws 629 +64.233.169.18 mail.google.com gws 643 +64.15.126.219 www.google.com gws 1375 +64.233.189.104 www.google.com gws 91 +64.233.171.164 *.googleadservices.com gws 639 +64.233.180.149 *.doubleclick.net gws 625 +85.182.250.241 google.com gws 1385 +85.182.250.242 google.com gws 1339 +208.117.231.247 google.com gws 1698 +85.182.250.247 google.com gws 913 +64.233.184.176 *.vp.video.l.google.com gws 561 +64.15.124.119 google.com gws 554 +64.15.119.133 www.google.com gws 495 +203.211.0.45 google.com gws 426 +64.233.185.142 *.au.doubleclick.net gws 595 +64.15.126.202 www.google.com gws 934 +208.117.224.162 www.google.com gws 954 +208.117.224.167 www.google.com gws 758 +208.117.230.250 google.com gws 1137 +64.15.113.181 google.com gws 2214 +64.233.166.167 *.googleadservices.com gws 674 +64.233.183.137 *.googlegroups.com gws 710 +64.233.189.81 sandbox.google.com gws 86 +203.165.13.231 google.com gws 375 +218.176.242.177 google.com gws 557 +111.168.255.45 google.com gws 499 +64.233.182.104 www.google.com gws 610 +208.117.228.178 google.com gws 609 +212.188.15.152 google.com gws 535 +212.188.15.153 google.com gws 597 +202.39.143.40 google.com gws 128 +208.117.253.20 google.com gws 638 +64.233.181.17 mail.google.com gws 786 +64.15.126.146 www.google.com gws 1383 +84.235.77.84 google.com gws 1607 +208.117.224.113 google.com gws 754 +106.162.192.162 google.com gws 221 +208.117.233.22 google.com gws 372 +173.194.124.15 google.com gws 950 +173.194.194.98 *.google.com gws 577 +64.233.180.99 www.google.com gws 624 +64.233.179.100 *.google.com gws 655 +210.242.125.25 google.com gws 95 +64.15.114.52 google.com gws 456 +86.127.118.158 google.com gws 971 +212.188.15.173 google.com gws 590 +64.15.114.57 google.com gws 622 +64.233.162.93 *.google.com gws 778 +210.153.73.113 google.com gws 781 +203.117.34.167 google.com gws 735 +64.15.117.54 google.com gws 993 +123.205.250.177 google.com gws 675 +64.233.171.86 *.orkut.com gws 684 +212.188.15.168 google.com gws 1423 diff --git a/goagent/3.1.32/local/google_ip.py b/goagent/3.1.32/local/google_ip.py index 153d4a99e7..850a68d299 100644 --- a/goagent/3.1.32/local/google_ip.py +++ b/goagent/3.1.32/local/google_ip.py @@ -14,11 +14,12 @@ import httplib import random import Queue +import math # const value: -max_check_ip_thread_num = 10 -min_good_ip_num = 95 -max_good_ip_num = 100 +max_check_ip_thread_num = 30 +min_good_ip_num = 100 +max_good_ip_num = 1000 timeout = 5000 # 5 second @@ -30,7 +31,7 @@ class Check_ip(): remove_ip_thread_num = 0 remove_ip_thread_lock = threading.Lock() - ip_dict = {} # ip_str => { 'handshake_time'=>?, 'domain'=>, 'server'=>?} + ip_dict = {} # ip_str => { 'handshake_time'=>?, 'domain'=>, 'server'=>?, 'timeout'=>} gws_ip_list = [] # gererate from ip_dict, sort by handshake_time, when get_batch_ip ip_lock = threading.Lock() iplist_need_save = 0 @@ -51,9 +52,10 @@ def __init__(self): self.search_more_google_ip() def load_ip(self): - try: - fd = open("good_ip.txt", "r") - for line in fd.readlines(): + with open("good_ip.txt", "r") as fd: + lines = fd.readlines() + for line in lines: + try: str_l = line.split(' ') if len(str_l) != 4: logging.warning("line err: %s", line) @@ -63,16 +65,18 @@ def load_ip(self): server = str_l[2] handshake_time = int(str_l[3]) - #logging.info("load ip: %s time:%d domain:%s server:%s", ip_str, handshake_time, domain, server) + logging.info("load ip: %s time:%d domain:%s server:%s", ip_str, handshake_time, domain, server) self.add_ip(ip_str, handshake_time, domain, server) - fd.close() - logging.info("load google iplist num: %d, gws num:%d", len(self.ip_dict), len(self.gws_ip_list)) + #logging.debug("load_ip ip:%s time:%d", ip_str, handshake_time) + except Exception as e: + logging.warn("load_ip line:%s err:%s", line, e) - p = threading.Thread(target = self.check_exist_ip) - p.daemon = True - p.start() - except Exception as e: - logging.warn("load google good ip fail: %s", e) + logging.info("load google iplist num: %d, gws num:%d", len(self.ip_dict), len(self.gws_ip_list)) + self.try_sort_ip_by_handshake_time(force=True) + + p = threading.Thread(target = self.check_exist_ip) + p.daemon = True + p.start() def save_ip_list(self): if self.iplist_need_save == 0: @@ -89,9 +93,58 @@ def save_ip_list(self): finally: self.ip_lock.release() + def try_sort_ip_by_handshake_time(self, force=False): + if time.time() - self.last_sort_time_for_gws < 10 and not force: + return + self.last_sort_time_for_gws = time.time() + + self.ip_lock.acquire() + try: + ip_dict_handshake_time = {} + for ip_str in self.ip_dict: + if 'gws' not in self.ip_dict[ip_str]['server']: + continue + ip_dict_handshake_time[ip_str] = self.ip_dict[ip_str]['handshake_time'] + + ip_time = sorted(ip_dict_handshake_time.items(), key=operator.itemgetter(1)) + self.gws_ip_list = [ip_str for ip_str,handshake_time in ip_time] + + + #self._clean_bad_ip_if_we_have_too_many_ips + finally: + self.ip_lock.release() + + def get_gws_ip(self): + self.try_sort_ip_by_handshake_time() + + self.ip_lock.acquire() + try: + ip_num = len(self.gws_ip_list) + if ip_num == 0: + #logging.warning("no gws ip") + time.sleep(1) + return None + + while True: + fastest_num = min(ip_num-1, 40) + #index = random.randint(0, fastest_num) + index = get_random_pr(fastest_num) + ip_str = self.gws_ip_list[index] + handshake_time = self.ip_dict[ip_str]["handshake_time"] + fail_time = self.ip_dict[ip_str]["fail_time"] + if time.time() - fail_time < 10: + continue + + logging.debug("get ip:%s t:%d", ip_str, handshake_time) + return ip_str + except Exception as e: + logging.error("get_gws_ip fail:%s", e) + finally: + self.ip_lock.release() + def add_ip(self, ip_str, handshake_time, domain=None, server=None): if not isinstance(ip_str, basestring): - logging.error("set_ip input") + logging.error("add_ip input") handshake_time = int(handshake_time) if handshake_time >= timeout: @@ -101,11 +154,12 @@ def add_ip(self, ip_str, handshake_time, domain=None, server=None): try: if ip_str in self.ip_dict: self.ip_dict[ip_str]['handshake_time'] = handshake_time + self.ip_dict[ip_str]['timeout'] = 0 #logging.debug("add ip:%s duplicated", ip_str) return False self.iplist_need_save = 1 - self.ip_dict[ip_str] = {'handshake_time':handshake_time, 'domain':domain, 'server':server} + self.ip_dict[ip_str] = {'handshake_time':handshake_time, 'domain':domain, 'server':server, 'timeout':0, "history":"%d,"% handshake_time, "fail_time":0} if 'gws' in server: self.gws_ip_list.append(ip_str) @@ -121,41 +175,60 @@ def update_ip(self, ip_str, handshake_time): logging.error("set_ip input") handshake_time = int(handshake_time) - if handshake_time >= timeout: - self.report_connect_fail(ip_str) + if handshake_time < 5: # this is impossible return self.ip_lock.acquire() try: if ip_str in self.ip_dict: - self.ip_dict[ip_str]['handshake_time'] = handshake_time + + # Case: some good ip, normal handshake time is 300ms + # some time ip lost cause handshake time become 2000ms + # this ip will no return back to first good ip list until all become bad + # There for, prevent handshake time increase too quickly. + org_time = self.ip_dict[ip_str]['handshake_time'] + if handshake_time - org_time > 200: + self.ip_dict[ip_str]['handshake_time'] = org_time + 200 + else: + self.ip_dict[ip_str]['handshake_time'] = handshake_time + + self.ip_dict[ip_str]['timeout'] = 0 + #self.ip_dict[ip_str]['history'] += "%d," % handshake_time + self.ip_dict[ip_str]["fail_time"] = 0 return #logging.debug("update ip:%s not exist", ip_str) except Exception as e: - logging.error("set_ip err:%s", e) + logging.error("update_ip err:%s", e) finally: self.ip_lock.release() - def report_connect_fail(self, ip_str): + def report_connect_fail(self, ip_str, force_remove=False): if not isinstance(ip_str, basestring): logging.error("set_ip input") if time.time() - self.network_fail_time < 3: return - + ip_removed = False self.ip_lock.acquire() try: if not ip_str in self.ip_dict: return - handshake_time = self.ip_dict[ip_str]['handshake_time'] - handshake_time += 300 - self.ip_dict[ip_str]['handshake_time'] = handshake_time + fail_time = self.ip_dict[ip_str]["fail_time"] + if time.time() - fail_time < 1: + return + + # increase handshake_time to make it can be used in lower probability + self.ip_dict[ip_str]['handshake_time'] += 200 + self.ip_dict[ip_str]['timeout'] += 1 + #self.ip_dict[ip_str]['history'] += "fail," + self.ip_dict[ip_str]["fail_time"] = time.time() - if handshake_time >= timeout: + if force_remove or self.ip_dict[ip_str]['timeout'] >= 5: + ip_removed = True property = self.ip_dict[ip_str] server = property['server'] del self.ip_dict[ip_str] @@ -174,7 +247,8 @@ def report_connect_fail(self, ip_str): finally: self.ip_lock.release() - self.try_sort_ip_by_handshake_time(force=True) + if ip_removed or True: + self.try_sort_ip_by_handshake_time(force=True) if not self.is_ip_enough(): self.search_more_google_ip() @@ -192,24 +266,11 @@ def try_remove_thread(self): p.daemon = True p.start() - def network_is_ok(self): - try: - conn = httplib.HTTPConnection("www.baidu.com") - conn.request("HEAD", "/") - response = conn.getresponse() - if response.status == 200: - return True - else: - return False - except: - return False - def remove_ip_process(self): try: - while True: if not self.network_is_ok(): - self.network_fail_time = time.time() + logging.warn("network is unreachable. check your network connection.") return try: @@ -225,49 +286,52 @@ def remove_ip_process(self): logging.info("real remove ip:%s ", ip_str) + #self.check_exist_ip() finally: self.remove_ip_thread_lock.acquire() self.remove_ip_thread_num -= 1 self.remove_ip_thread_lock.release() + def network_is_ok(self): + if time.time() - self.network_fail_time < 3: + return False - def try_sort_ip_by_handshake_time(self, force=False): - self.ip_lock.acquire() try: - if force or time.time() - self.last_sort_time_for_gws > 10: - ip_dict_handshake_time = {} - for ip_str in self.ip_dict: - if 'gws' not in self.ip_dict[ip_str]['server']: - continue - ip_dict_handshake_time[ip_str] = self.ip_dict[ip_str]['handshake_time'] - ip_time = sorted(ip_dict_handshake_time.items(), key=operator.itemgetter(1)) - self.gws_ip_list = [ip_str for ip_str,handshake_time in ip_time] - self.last_sort_time_for_gws = time.time() - finally: - self.ip_lock.release() + conn = httplib.HTTPSConnection("github.com", 443) + header = {"user-agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Safari/537.36", + "accept":"application/json, text/javascript, */*; q=0.01", + "accept-encoding":"gzip, deflate, sdch", + "accept-language":'en-US,en;q=0.8,ja;q=0.6,zh-CN;q=0.4,zh;q=0.2', + "connection":"keep-alive" + } + conn.request("HEAD", "/", headers=header) + response = conn.getresponse() + if response.status: + return True + except: + pass + self.network_fail_time = time.time() + return False + def _clean_bad_ip_if_we_have_too_many_ips(self): + ip_num = len(self.gws_ip_list) + if ip_num < max_good_ip_num: + return - def get_gws_ip(self): - self.try_sort_ip_by_handshake_time() + for i in range(ip_num-1, min_good_ip_num, -1): + ip = self.gws_ip_list[i] + property = self.ip_dict[ip] - self.ip_lock.acquire() - try: - ip_num = len(self.gws_ip_list) - if ip_num == 0: - #logging.warning("no gws ip") - time.sleep(1) - return None + if property['handshake_time'] < 600: + return + + server = property['server'] + del self.ip_dict[ip] + + if 'gws' in server: + self.gws_ip_list.remove(ip) - fastest_num = min(ip_num-1, 5) - index = random.randint(0, fastest_num) - ip_str = self.gws_ip_list[index] - #logging.debug("get ip:%s t:%d", ip_str, handshake_time) - return ip_str - except Exception as e: - logging.error("get_gws_ip fail:%s", e) - finally: - self.ip_lock.release() def check_ip(self, ip_str): @@ -276,13 +340,14 @@ def check_ip(self, ip_str): return False if self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type): - logging.info("add %s CN:%s type:%s time:%d gws:%d ", ip_str, - result.domain, result.server_type, result.handshake_time, len(self.gws_ip_list)) + #logging.info("add %s CN:%s type:%s time:%d gws:%d ", ip_str, + # result.domain, result.server_type, result.handshake_time, len(self.gws_ip_list)) + logging.info("check_ip add ip:%s time:%d", ip_str, result.handshake_time) return True def runJob(self): - while not self.is_ip_enough(): + while True:# not self.is_ip_enough(): try: time.sleep(1) ip_int = ip_range.get_ip() @@ -306,21 +371,28 @@ def search_more_google_ip(self): p.start() def check_exist_ip(self): + if not self.network_is_ok(): + logging.warn("check_exist_ip network is fail, check your network connection.") + return + self.ip_lock.acquire() tmp_ip_list = [x for x in self.gws_ip_list] self.ip_lock.release() for ip_str in tmp_ip_list: - result = check_ip.test(ip_str) - if not result or not result.appspot_ok: - self.report_connect_fail(ip_str) + result = check_ip.test_gws(ip_str) + if not result: + logging.info("check_exist_ip fail ip:%s ", ip_str) + self.report_connect_fail(ip_str, force_remove=True) else: self.update_ip(ip_str, result.handshake_time) + logging.info("check_exist_ip update ip:%s server:%s time:%d", ip_str, result.server_type, result.handshake_time) self.save_ip_list() -google_ip = Check_ip() +if __name__ != "__main__": + google_ip = Check_ip() def test(): check = Check_ip() @@ -342,12 +414,37 @@ def test_profile(): pr.disable() pr.print_stats(sort="cum") - -if __name__ == '__main__': +def test_network(): check = Check_ip() res = check.network_is_ok() print res +def get_random_pr(num): + n0 = num + r = 2 + + for i in range(r): + n0 = n0 * n0 + + n1 = random.randint(0, n0) + for i in range(r): + n1 = math.sqrt(n1) + + n2 = num - n1 + return int(n2 ) + +def test_random(): + result = {} + for i in range(300): + v = get_random_pr(30) + if not v in result: + result[v] = 0 + result[v] += 1 + for k in result: + print k, result[k] + +if __name__ == '__main__': + test_random() # test cast # 1. good_ip.txt not exist when startup, auto scan good ip, then save # 2. good_ip.txt exist, load ip list, and check it. diff --git a/goagent/3.1.32/local/proxy.ini b/goagent/3.1.32/local/proxy.ini index ae35ed269f..028c795a01 100644 --- a/goagent/3.1.32/local/proxy.ini +++ b/goagent/3.1.32/local/proxy.ini @@ -5,7 +5,7 @@ visible = 1 debuginfo = 0 [gae] -appid = xxnet-1|xxnet-2|xxnet-3|xxnet-4|xxnet-5|xxnet-6|xxnet-7|xxnet-8|xxnet-9|xxnet-10|xxnet-20|xxnet-21|xxnet-22|xxnet-23|xxnet-24|xxnet-25|xxnet-26|xxnet-27|xxnet-28|xxnet-29|xxnet-30|xxnet-31|xxnet-32|xxnet-33|xxnet-34|xxnet-35|xxnet-36|xxnet-37|xxnet-38|xxnet-39|xxnet-40|xxnet-41|xxnet-42|xxnet-43|xxnet-44 +appid = xxnet-1|xxnet-2|xxnet-3|xxnet-4|xxnet-5|xxnet-6|xxnet-7|xxnet-8|xxnet-9|xxnet-10|xxnet-11|xxnet-12|xxnet-13|xxnet-14|xxnet-15|xxnet-16|xxnet-17|xxnet-18|xxnet-19|xxnet-20|xxnet-21|xxnet-22|xxnet-23|xxnet-24|xxnet-25|xxnet-26|xxnet-27|xxnet-28|xxnet-29|xxnet-30|xxnet-31|xxnet-32|xxnet-33|xxnet-34|xxnet-35|xxnet-36|xxnet-37|xxnet-38|xxnet-39|xxnet-40|xxnet-41|xxnet-42|xxnet-43|xxnet-44|xxnet-45|xxnet-46|xxnet-47|xxnet-48|xxnet-49|xxnet-50|xxnet-51|xxnet-52|xxnet-53|xxnet-54|xxnet-55|xxnet-56|xxnet-57|xxnet-58|xxnet-59 password = path = /2 ;appid = fyxiangjiao170|fyxiangjiao171|fyxiangjiao172|fyxiangjiao173|fyxiangjiao174|fyxiangjiao175|fyxiangjiao176|fyxiangjiao177|fyxiangjiao178|fyxiangjiao179|fyxiangjiao190|fyxiangjiao191|fyxiangjiao192|fyxiangjiao193|fyxiangjiao194|fyxiangjiao195|fyxiangjiao196|fyxiangjiao199|fyxiangjiao199|fyxiangjiao199|fyxiangjiao210|fyxiangjiao211|fyxiangjiao212|fyxiangjiao213|fyxiangjiao214|fyxiangjiao215|fyxiangjiao216|fyxiangjiao217|fyxiangjiao218|fyxiangjiao219|fyxiangjiao220|fyxiangjiao221|fyxiangjiao222|fyxiangjiao223|fyxiangjiao224|fyxiangjiao225|fyxiangjiao226|fyxiangjiao227|fyxiangjiao228|fyxiangjiao229|fyxiangjiao230|fyxiangjiao231|fyxiangjiao232|fyxiangjiao233|fyxiangjiao234|fyxiangjiao235|fyxiangjiao236|fyxiangjiao237|fyxiangjiao238|fyxiangjiao239|fyxiangjiao240|fyxiangjiao241|fyxiangjiao242|fyxiangjiao243|fyxiangjiao244|fyxiangjiao245|fyxiangjiao246|fyxiangjiao247|fyxiangjiao248|fyxiangjiao249|fyxiangjiao250|fyxiangjiao251|fyxiangjiao252|fyxiangjiao253|fyxiangjiao254|fyxiangjiao255|fyxiangjiao256|fyxiangjiao257|fyxiangjiao258|fyxiangjiao259|fyxiangjiao260|fyxiangjiao261|fyxiangjiao262|fyxiangjiao263|fyxiangjiao264|fyxiangjiao265|fyxiangjiao266|fyxiangjiao267|fyxiangjiao268|fyxiangjiao269|fyxiangjiao270|fyxiangjiao271|fyxiangjiao272|fyxiangjiao273|fyxiangjiao274|fyxiangjiao275|fyxiangjiao276|fyxiangjiao277|fyxiangjiao278|fyxiangjiao279|fyxiangjiao280|fyxiangjiao281|fyxiangjiao282|fyxiangjiao283|fyxiangjiao284|fyxiangjiao285|fyxiangjiao286|fyxiangjiao287|fyxiangjiao288|fyxiangjiao83|fyxiangjiao290|fyxiangjiao291|fyxiangjiao292|fyxiangjiao293|fyxiangjiao294|fyxiangjiao295|fyxiangjiao296|fyxiangjiao297|fyxiangjiao298|fyxiangjiao299|fyxiangjiao300|fyxiangjiao301|fyxiangjiao302|fyxiangjiao303|fyxiangjiao304|fyxiangjiao305|fyxiangjiao306|fyxiangjiao307|fyxiangjiao308|fyxiangjiao309|fyxiangjiao310|fyxiangjiao311|fyxiangjiao312|fyxiangjiao313|fyxiangjiao314|fyxiangjiao315|fyxiangjiao316|fyxiangjiao317|fyxiangjiao318|fyxiangjiao319|fyxiangjiao320|fyxiangjiao321|fyxiangjiao322|fyxiangjiao323|fyxiangjiao324|fyxiangjiao325|fyxiangjiao326|fyxiangjiao327|fyxiangjiao328|fyxiangjiao329|fyxiaojiao350|fyxiangjiao351|fyxiangjiao352|fyxiangjiao353|fyxiangjiao354|fyxiangjiao355|fyxiangjiao356|fyxiangjiao357|fyxiangjiao358|fyxiangjiao359|chinacrosswall060|chinacrosswall061|chinacrosswall063|chinacrosswall064|chinacrosswall065|chinacrosswall066|chinacrosswall067|chinacrosswall068|chinacrosswall069|chinacrosswall070|chinacrosswall071|chinacrosswall072|chinacrosswall073|chinacrosswall074|chinacrosswall075|chinacrosswall076|chinacrosswall077|chinacrosswall078|chinacrosswall079|chinacrosswall050|chinacrosswall051|chinacrosswall052|chinacrosswall053|chinacrosswall054|chinacrosswall055|chinacrosswall056|chinacrosswall057|chinacrosswall058|chinacrosswall059|chinacrosswall040|chinacrosswall041|chinacrosswall042|chinacrosswall043|chinacrosswall044|chinacrosswall045|chinacrosswall046|chinacrosswall047|chinacrosswall048|chinacrosswall049|chinacrosswall030|chinacrosswall031|chinacrosswall032|chinacrosswall033|chinacrosswall034|chinacrosswall035|chinacrosswall036|chinacrosswall037|chinacrosswall038|chinacrosswall039|chinacrosswall090|chinacrosswall091|chinacrosswall092|chinacrosswall093|chinacrosswall094|chinacrosswall095|chinacrosswall096|chinacrosswall097|chinacrosswall098|chinacrosswall099|chinacrosswall100|chinacrosswall101|chinacrosswall102|chinacrosswall103|chinacrosswall104|chinacrosswall105|chinacrosswall106|chinacrosswall107|chinacrosswall108|chinacrosswall109|chinacrosswall120|chinacrosswall121|chinacrosswall122|chinacrosswall123|chinacrosswall124|chinacrosswall125|chinacrosswall126|chinacrosswall127|chinacrosswall128|chinacrosswall129|chinacrosswall130|chinacrosswall131|chinacrosswall132|chinacrosswall133|chinacrosswall134|chinacrosswall135|chinacrosswall136|chinacrosswall137|chinacrosswall138|chinacrosswall139|chinacrosswall000|chinacrosswall001|chinacrosswall002|chinacrosswall003|chinacrosswall004|chinacrosswall005|chinacrosswall006|chinacrosswall007|chinacrosswall008|chinacrosswall009|chinacrosswall010|chinacrosswall011|chinacrosswall012|chinacrosswall013|chinacrosswall014|chinacrosswall015|chinacrosswall016|chinacrosswall017|chinacrosswall018|chinacrosswall019|chinacrosswall020|chinacrosswall021|chinacrosswall022|chinacrosswall023|chinacrosswall024|chinacrosswall025|chinacrosswall026|chinacrosswall027|chinacrosswall028|chinacrosswall029|chinacrosswall110|chinacrosswall111|chinacrosswall112|chinacrosswall113|chinacrosswall114|chinacrosswall115|chinacrosswall116|chinacrosswall117|chinacrosswall118|chinacrosswall119 @@ -31,9 +31,9 @@ gp4.googleusercontent.com = gws gp5.googleusercontent.com = gws gp6.googleusercontent.com = gws themes.googleusercontent.com = gws -talk.google.com = -talk.l.google.com = -talkx.l.google.com = +talk.google.com = gws +talk.l.google.com = gws +talkx.l.google.com = gws .google.com = gws .ytimg.com = gws .doubleclick.net = gws diff --git a/goagent/3.1.32/local/proxy_handler.py b/goagent/3.1.32/local/proxy_handler.py index b08504215e..afbf63b8fe 100644 --- a/goagent/3.1.32/local/proxy_handler.py +++ b/goagent/3.1.32/local/proxy_handler.py @@ -26,6 +26,7 @@ from cert_util import CertUtil from connect_manager import https_manager,forwork_manager +import traceback current_path = os.path.dirname(os.path.abspath(__file__)) python_path = os.path.abspath( os.path.join(current_path, os.pardir, os.pardir, os.pardir, 'python27', '1.0')) @@ -137,11 +138,17 @@ def gae_urlfetch(method, url, headers, payload, app_server, **kwargs): request_headers['Content-Length'] = str(len(payload)) # post data + #start_time = time.time() response = https_manager.request(request_method, app_server, payload, request_headers) + #stop_time = time.time() + #cost_time = stop_time - start_time + if hasattr(response, "status"): response.app_status = response.status else: return response + #logging.debug("request time:%d status:%d url:%s ", int(cost_time * 1000), response.status, url) + response.app_options = response.getheader('X-GOA-Options', '') if response.status != 200: return response @@ -341,7 +348,6 @@ class GAEProxyHandler(BaseHTTPServer.BaseHTTPRequestHandler): normcookie = functools.partial(re.compile(', ([^ =]+(?:=|$))').sub, '\\r\\nSet-Cookie: \\1') normattachment = functools.partial(re.compile(r'filename=(.+?)').sub, 'filename="\\1"') - appid = appid_manager.get_appid() def setup(self): @@ -423,9 +429,12 @@ def do_AGENT(self): # TODO: test validate = 1 kwargs['validate'] = 0 - appid = self.appid + appid = appid_manager.get_appid() app_server = "https://" + appid + ".appspot.com/2?" + time_start = time.time() response = gae_urlfetch(self.command, self.path, request_headers, payload, app_server, **kwargs) + time_stop = time.time() + time_cost = int((time_stop - time_start) * 1000) if not response: if retry >= config.FETCHMAX_LOCAL-1: html = generate_message_html('502 URLFetch failed', 'Local URLFetch %r failed' % self.path, str(errors)) @@ -440,9 +449,9 @@ def do_AGENT(self): if response.app_status == 404: logging.warning('APPID %r not exists, remove it.', appid) appid_manager.report_not_exist(appid) - self.appid = appid_manager.get_appid() + appid = appid_manager.get_appid() - if not self.appid: + if not appid: html = generate_message_html('404 No usable Appid Exists', 'No usable Appid Exists, please add appid') self.wfile.write(b'HTTP/1.0 502\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8')) response.close() @@ -455,9 +464,9 @@ def do_AGENT(self): if response.app_status == 503: logging.warning('APPID %r out of auota, remove it.', appid) appid_manager.report_out_of_quota(appid) - self.appid = appid_manager.get_appid() + appid = appid_manager.get_appid() - if not self.appid: + if not appid: html = generate_message_html('404 No usable Appid Exists', 'No usable Appid Exists, please add appid') self.wfile.write(b'HTTP/1.0 502\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8')) response.close() @@ -481,7 +490,7 @@ def do_AGENT(self): # first response, has no retry. if not headers_sent: - logging.info('"GAE %s %s HTTP/1.1" status:%s len:%s', self.command, self.path, response.status, response.getheader('Content-Length', '-')) + logging.info('"GAE t:%d %s %s HTTP/1.1" status:%s len:%s', time_cost, self.command, self.path, response.status, response.getheader('Content-Length', '-')) if response.status == 206: # 206 means "Partial Content" fetchservers = [app_server] @@ -532,6 +541,7 @@ def do_AGENT(self): return else: logging.exception('GAEProxyHandler.do_METHOD_AGENT %r return %r, errno: %d ', self.path, e, int(e.args[0])) #IOError(9, 'Bad file descriptor') + traceback.print_exc() def do_CONNECT(self): """handle CONNECT cmmand, socket forward or deploy a fake cert""" @@ -555,7 +565,7 @@ def do_CONNECT_FWD(self): for i in range(3): try: - if host == "appengine.google.com": + if host == "appengine.google.com" or host == "www.google.com": connected_in_s = 5 # goagent upload to appengine is slow, it need more 'fresh' connection. @@ -569,7 +579,7 @@ def do_CONNECT_FWD(self): break elif i == 0: # only logging first create_connection error - logging.error('http_util.create_connection((host=%r, port=%r)) timeout', host, port, ) + logging.warn('http_util.create_connection((host=%r, port=%r)) timeout', host, port, ) except NetWorkIOError as e: if e.args[0] == 9: logging.error('GAEProxyHandler direct forward remote (%r, %r) failed', host, port) diff --git a/goagent/3.1.32/local/remote_control.py b/goagent/3.1.32/local/remote_control.py index c7bace6ff4..e5d69fcfa5 100644 --- a/goagent/3.1.32/local/remote_control.py +++ b/goagent/3.1.32/local/remote_control.py @@ -31,6 +31,8 @@ def do_GET(self): return self.req_log_handler() elif path == "/status": return self.req_status_handler() + elif path == "/ip_list": + return self.req_ip_list_handler() elif path == "/quit": config.keep_run = False data = "Quit" @@ -111,12 +113,18 @@ def req_status_handler(self): reqs = urlparse.parse_qs(req, keep_blank_values=True) data = '' + if "user-agent" in self.headers.dict: + user_agent = self.headers.dict["user-agent"] + else: + user_agent = "" + gws_ip_num = len(google_ip.gws_ip_list) res_arr = {"gws_ip_num": gws_ip_num, "sys_platform":sys.platform, "os_system":platform.system(), "os_version":platform.version(), "os_release":platform.release(), + "browser":user_agent, "goagent_version": config.__version__, "python_version": config.python_version, "proxy_listen":config.LISTEN_IP + ":" + str(config.LISTEN_PORT), @@ -128,3 +136,16 @@ def req_status_handler(self): self.wfile.write(('HTTP/1.1 200\r\nAccess-Control-Allow-Origin: *\r\nContent-Type: %s\r\nContent-Length: %s\r\n\r\n' % (mimetype, len(data))).encode()) self.wfile.write(data) + def req_ip_list_handler(self): + data = "" + for ip in google_ip.gws_ip_list: + handshake_time = google_ip.ip_dict[ip]["handshake_time"] + timeout = google_ip.ip_dict[ip]["timeout"] + history = google_ip.ip_dict[ip]["history"] + data += "%s \t %d \t %d \t %s\r\n" % (ip, handshake_time, timeout, history) + + + mimetype = 'text/plain' + self.wfile.write(('HTTP/1.1 200\r\nAccess-Control-Allow-Origin: *\r\nContent-Type: %s\r\nContent-Length: %s\r\n\r\n' % (mimetype, len(data))).encode()) + self.wfile.write(data) + diff --git a/launcher/1.0.0/config.py b/launcher/1.0.0/config.py index 4dfb7e8559..a8f2223fc1 100644 --- a/launcher/1.0.0/config.py +++ b/launcher/1.0.0/config.py @@ -33,11 +33,50 @@ def save(): except Exception as e: logging.warn("save config %s fail %s", data_path, e) +def get(path, default_val=""): + global config + try: + value = default_val + cmd = "config" + for p in path: + cmd += '["%s"]' % p + value = eval(cmd) + return value + except: + return default_val + +def _set(m, k_list, v): + k0 = k_list[0] + if len(k_list) == 1: + m[k0] = v + return + if k0 not in m: + m[k0] = {} + _set(m[k0], k_list[1:], v) + +def set(path, val): + global config + _set(config, path, val) + + def main(): load() #config["tax"] = 260 #save() print yaml.dump(config) +def test(): + load() + val = get(["web_ui", "popup_webui"], 0) + print val + +def test2(): + set(["web_ui", "popup_webui"], 0) + set(["web_ui", "popup"], 0) + print config + if __name__ == "__main__": - main() \ No newline at end of file + test2() + #main() + #a = eval('2*3') + #eval("conf = {}") \ No newline at end of file diff --git a/launcher/1.0.0/html/block/config.html b/launcher/1.0.0/html/block/config.html index 6f0858759c..1e3dceee9a 100644 --- a/launcher/1.0.0/html/block/config.html +++ b/launcher/1.0.0/html/block/config.html @@ -1,10 +1,15 @@

-

Check update: +

Check update:
-

+ +

Popup Web UI when startup: +
+ +
+



@@ -14,27 +19,27 @@

Check update: \ No newline at end of file diff --git a/launcher/1.0.0/html/block/goagent/config.html b/launcher/1.0.0/html/block/goagent/config.html index 1df90778d7..6bf6b7542a 100644 --- a/launcher/1.0.0/html/block/goagent/config.html +++ b/launcher/1.0.0/html/block/goagent/config.html @@ -7,6 +7,8 @@

GoAgent config

+
Register google account and apply AppEngine ip: Help Link
+