from urllib import parse as url_parse

from logger import crawler
from .workers import app
from page_get import get_page
from config import get_max_search_page
from page_parse import search as parse_search
from db.dao import (
    KeywordsOper, KeywordsDataOper, WbDataOper)


# This url is just for original weibos.
# If you want other kind of search, you can change the url below
URL = 'http://s.weibo.com/weibo/{}&scope=ori&suball=1&page={}'
LIMIT = get_max_search_page() + 1


@app.task(ignore_result=True)
def search_keyword(keyword, keyword_id):
    crawler.info('We are searching keyword "{}"'.format(keyword))
    cur_page = 1
    encode_keyword = url_parse.quote(keyword)
    while cur_page < LIMIT:
        cur_url = URL.format(encode_keyword, cur_page)
        # current only for login, maybe later crawling page one without login
        search_page = get_page(cur_url, auth_level=2)
        if not search_page:
            crawler.warning('No search result for keyword {}, the source page is {}'.format(keyword, search_page))
            return

        search_list = parse_search.get_search_info(search_page)

        # Because the search results are sorted by time, if any result has been stored in mysql,
        # we need not crawl the same keyword in this turn
        for wb_data in search_list:
            rs = WbDataOper.get_wb_by_mid(wb_data.weibo_id)
            if rs:
                crawler.info('Keyword {} has been crawled in this turn'.format(keyword))
                return
            else:
                WbDataOper.add_one(wb_data)
                KeywordsDataOper.insert_keyword_wbid(keyword_id, wb_data.weibo_id)
                # send task for crawling user info
                app.send_task('tasks.user.crawl_person_infos', args=(wb_data.uid,), queue='user_crawler',
                              routing_key='for_user_info')
        if cur_page == 1:
            cur_page += 1
        elif 'page next S_txt1 S_line1' in search_page:
            cur_page += 1
        else:
            crawler.info('Keyword {} has been crawled in this turn'.format(keyword))
            return


@app.task(ignore_result=True)
def execute_search_task():
    keywords = KeywordsOper.get_search_keywords()
    for each in keywords:
        app.send_task('tasks.search.search_keyword', args=(each[0], each[1]), queue='search_crawler',
                      routing_key='for_search_info')
@@ -1360,32 +1360,162 @@
 _data.weibo_id)%0A
+            KeywordsDataOper.insert_keyword_wbid(keyword_id, wb_data.weibo_id)%0A            # todo incremental crawling using time%0A
             if r
@@ -1539,39 +1539,37 @@
   crawler.info('
-Keyword
+Weibo
  %7B%7D has been cra
@@ -1576,36 +1576,42 @@
 wled
- in this turn'.format(keywor
+, skip it.'.format(wb_data.weibo_i
 d))%0A
@@ -1626,22 +1626,24 @@
         
-return
+continue
 %0A       
@@ -1701,91 +1701,8 @@
 ta)%0A
-                KeywordsDataOper.insert_keyword_wbid(keyword_id, wb_data.weibo_id)%0A
     
@@ -1983,33 +1983,25 @@
 if '
-page next S_txt1 S_line1'
+noresult_tit' not
  in 

fix bug of incomplete search result crawling