gevent超时后用curl做补偿

Author: huhao

python/goodorbad.py

@@ -13,7 +13,7 @@
         ]
 
 def sum1(input):
-    return sum((lambda x: (x < 10 and [x*2] or [x/2])[0])(num)
+    return sum((lambda x: x < 10 and x*2 or x/2)(num)
             for seq in inputdata 
             if len(seq) >= 5
             for num in seq

python/monitor_web_all.py

@@ -1,3 +1,7 @@
+#encoding=utf-8
+'''
+先用gevent跑，然后把超时的用curl重试
+'''
 from gevent import spawn,joinall,sleep
 from gevent.pool import Pool
 from monitor_web_gevent import process_results, curl as gevent_curl
@@ -52,15 +56,16 @@ def init_log(ips):
         with open(log_path, 'a') as f:
             f.write(ip)
 
-def log_result(result):
-    ip = result[1]
-    log_path = './log/%s' % (ip,)
-    with open(log_path, 'a') as f:
-        try:
-            int(result[0])
-            f.write('*')
-        except:
-            f.write('#')
+def log_results(results):
+    for result in results:
+        ip = result[1]
+        log_path = './log/%s' % (ip,)
+        with open(log_path, 'a') as f:
+            try:
+                int(result[0])
+                f.write('*')
+            except:
+                f.write('#')
 
 def end_log(ips):
  for ip in ips:
@@ -72,16 +77,15 @@ def process_ips(ips):
     all_results = get_results_use_pool()
     timeout_ips = get_timeout_ips(all_results)
     retry_results = retry_timeout_ips(timeout_ips)
-    merged_results = merge_results(retry_results, all_results)
-    process_results(merged_results)
-    for result in merged_results:
-        log_result(result)
+    return merge_results(retry_results, all_results)
 
 if __name__ == '__main__':
     iplist = getiplist()
     init_log(iplist)
     for i in range(100):
-        process_ips(iplist)
+        results = process_ips(iplist)
+        process_results(results)
+        log_results(results)
         print 'sleep.....'
         sleep(30)
     end_log(iplist)

python/monitor_web_curl.py

@@ -11,6 +11,29 @@
 CONNECT_TIMEOUT,DATA_TIMEOUT = 10, 10
 IP_COUNT, POOL_SIZE = 1000, 100
 
+def parse_stdout(stdout):
+    if not stdout:return None
+    lineend_index = stdout.find('\n')
+    if lineend_index < 0: return None
+    firstline = stdout[0: lineend_index]
+    match = regex_code.search(firstline)
+    if match:
+        return match.group(1)
+    return None
+
+def parse_stderr(stderr):
+    if not stderr:return None
+    match = regex_error.search(stderr)
+    if match:
+        return match.group(1)
+    return None
+
+
+def parse_curl_info(stdout, stderr):
+    info = parse_stdout(stdout)
+    if info: return info
+    return parse_stderr(stderr)
+
 def curl(ip):
     '''
     调用curl并获取结果，从结果里提取http应答码
@@ -20,17 +43,9 @@ def curl(ip):
             % (CONNECT_TIMEOUT, DATA_TIMEOUT, ip)
     process = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
 
-    info = process.stdout.readline().rstrip('\r\n')
-    info = regex_code.search(info)
-    if info:
-        info = info.group(1)
-
-    if not info:
-        info = process.stderr.read()
-        info = regex_error.search(info)
-        if info:
-            info= info.group(1)
-
+    stdout = process.stdout.read()
+    stderr = process.stderr.read()
+    info = parse_curl_info(stdout, stderr)
     print  info, ip
     return info, ip
 

python/monitor_web_gevent.py

@@ -4,7 +4,7 @@
 探测一组IP的可访问性
 '''
 from gevent import joinall,Timeout
-from gevent import monkey;monkey.patch_all()
+from gevent import monkey;monkey.patch_all(thread=False)
 from gevent.pool import Pool
 import urllib2
 import socket

python/popentimeout.py

@@ -22,22 +22,41 @@
 from subprocess import Popen, PIPE
 from gevent.socket import wait_read
 from sys import exc_info
-from traceback import format_exception
+try:
+    from errno import EBADF
+except ImportError:
+    EBADF = 9
 
 def runwithtimeout(cmd, timeout):
     process = Popen(cmd,shell=True, stdout=PIPE, stderr=PIPE,
             close_fds=True)
-    f = process.stdout
+    stdout, stderr = None, None
     try:
-        wait_read(f.fileno(), timeout)
-        return f.readlines()
-    except:
+        stdout = read(process.stdout, timeout)
+        stderr = read(process.stderr, timeout)
+    except Exception, ex:
+        print 'popen timeout:', cmd
+    finally:
         process.kill()
-        raise
+        return stdout, stderr
 
+def read(fromfile, timeout):
+    while True:
+        try:
+            wait_read(fromfile.fileno(), timeout)
+            return fromfile.read()
+        except:
+            ex = exc_info()[1]
+            if ex.args[0] == EBADF:
+                return ''
+            raise
+   
 if __name__ == '__main__':
     from gevent import spawn
-    task = spawn(runwithtimeout, 'whois baidu.com', 1)
+    from traceback import format_exception
+    cmd = 'curl -I --connect-timeout %s -m %s %s ' \
+            % (5, 5, '202.99.160.68')
+    task = spawn(runwithtimeout, cmd, 10)
     try:
         print ''.join(task.get())
     except: