@@ -0,0 +1,1876 @@
+#coding:utf-8%0Aimport gevent.monkey%0A%0Agevent.monkey.patch_socket()%0A%0Afrom gevent.pool import Pool%0Afrom gevent.subprocess import Popen, PIPE%0Aimport os%0Aimport sys%0Aimport gevent%0Afrom pyquery import PyQuery as pq%0A%0APOOL_NUM = 2%0ABASE_DIR = os.path.dirname(os.path.abspath(__file__))%0ASTOCK_FILE = os.path.join(BASE_DIR, %22stock.txt%22)%0ACOMPANY_DIR = os.path.join(BASE_DIR, 'company')%0ACOMPANY_URL = 'http://xueqiu.com/S/'%0A%0Apool = Pool(POOL_NUM)%0A%0A%0Adef judgeMarket(code):%0A    if str(code).startswith('6'):%0A        return 'SH'%0A    else:%0A        return 'SZ'%0A%0Adef crawl_company(code):%0A    m = judgeMarket(code)%0A    print 'start process %25s%25s' %25 (m, code)%0A    target_file = os.path.join(COMPANY_DIR, '%25s.csv' %25 code)%0A    if os.path.exists(target_file):%0A        print '%25s company file already exists' %25 code%0A    else:%0A        url = '%25s%25s%25s' %25 (COMPANY_URL, m, code)%0A        print 'crawl url : %25s' %25 url%0A        try:%0A            req = urllib2.Request(url)%0A            resp = urllib2.urlopen(req, timeout=10)%0A            status = resp.code%0A            content = resp.read()%0A            print 'crawl %25s status: %25s' %25 (code, status)%0A            pool.spawn(extract, target_file, content)%0A        except Exception, e:%0A            print 'crawl %25s err: %25s' %25 (code, str(e))%0A%0A%0Adef extract(target_file, data):%0A    pass%0A%0Adef clean_empty():%0A    for f in os.listdir(COMPANY_DIR):%0A        if f.endswith('.csv') and os.path.getsize(os.path.join(COMPANY_DIR, f)) %3C 10:%0A            print 'clean emtpy file ', f%0A            os.remove(os.path.join(COMPANY_DIR, f))%0A%0Adef run():%0A    with open(STOCK_FILE, 'rb') as f:%0A        for code in f:%0A            pool.spawn(crawl_company, code.strip())%0A    pool.join()%0A    clean_empty()%0A%0A%0Aif '__main__' == __name__:%0A    import time%0A    start_at = time.time()%0A    print 'begin at: %25s' %25 start_at%0A    run()%0A    print 'end at: %25s and used %25s' %25 (time.time(), time.time() - start_at)%0A

add crawl xueqiu