__author__ = 'mengpeng'
import eventlet
import ast
import urllib
import socket
from eventlet.green import urllib2
from pycrawler.utils.tools import fullstamp
from pycrawler.utils.tools import datestamp
from pycrawler.exception import ScraperException


def parseurl(url):
    data = None
    if '<args>' in url:
        parts = url.split('<args>')
        if len(parts) != 2:
            raise ScraperException('Wrong post url format: '+url)
        url = parts[0]
        try:
            data = ast.literal_eval(parts[1])
        except ValueError:
            raise ScraperException('Wrong post args format: '+parts[1])
    return url, data


def encodeurl(method, url, data=None):
    if method == 'POST' and data:
        url += '<args>' + data.__str__()
        return url
    elif method == 'GET' and data:
        url += urllib.urlencode(data)
        return url
    elif not data:
        return url
    else:
        raise ScraperException('Unsupported http method: '+method)


class Scraper(object):
    Dict = {}

    def __init__(self, spider):
        pass

    @staticmethod
    def register(cls):
        if isinstance(cls, type):
            Scraper.Dict[cls.__name__] = cls
            return cls
        else:
            raise ScraperException('Must register scraper with class name')

    @staticmethod
    def getScraper(name):
        if name in Scraper.Dict:
            return Scraper.Dict[name]
        else:
            raise ScraperException('No scraper class named '+name)

    def setargs(self, args):
        raise NotImplementedError

    def fetch(self, *args):
        raise NotImplementedError


@Scraper.register
class DefaultScraper(Scraper):

    def __init__(self, spider):
        super(DefaultScraper, self).__init__(spider)
        self._spider = spider
        self.args = {'debug': True,
                     'log': True,
                     'logfile': spider.name+'Scraper-'+datestamp()+'.log',
                     'timeout': 10}

    def setargs(self, args):
        if not isinstance(args, dict):
            raise ScraperException('Args must be a dict')
        for key, value in args.iteritems():
            self.args[key] = value

    def fetchone(self, oriurl):
        url, data = DefaultScraper.parseurl(oriurl)
        data = urllib.urlencode(data) if data else data
        try:
            res = urllib2.urlopen(url=url, data=data, timeout=self.args['timeout'])
        except (socket.timeout, urllib2.HTTPError, IOError) as e:
            self._debug(e.message+': '+oriurl)
            html = None
        else:
            html = res.read()
            res.close()
        if self.args['debug']:
            print('{0} [{1}] Scraped: {2}'.format(fullstamp(), self._spider.name, url))
        return oriurl, html

    def fetch(self, urllist):
        results = {}
        pool = eventlet.GreenPool()
        for url, html in pool.imap(self.fetchone, urllist):
            if html:
                results[url] = html
        return results

    def _debug(self, s):
        message = '{0} [{1}] {2}'.format(fullstamp(), self._spider.name, s)
        if self.args['debug']:
            print(message)

    @staticmethod
    def parseurl(url):
        return parseurl(url)

    @staticmethod
    def encodeurl(method, url, data=None):
        return encodeurl(method, url, data=data)
@@ -1806,138 +1806,71 @@
 elf.
-args = %7B'debug': True,%0A                     'log': True,%0A                     'logfile': spider.name+'Scraper-'+datestamp()+'.log'
+logger = self._spider.logger%0A        self.args = %7B'debug': True
 ,%0A  
@@ -2362,17 +2362,16 @@
  except 
-(
 socket.t
@@ -2376,18 +2376,148 @@
 .timeout
-,
  
+as e:%0A            self.logger.warning(self._spider.name+'-Scraper', e.message+': '+oriurl)%0A            html = None%0A        except (
 urllib2.
@@ -2535,21 +2535,16 @@
 IOError)
- as e
 :%0A      
@@ -2558,26 +2558,59 @@
 elf.
-_debug(e.message+'
+logger.error(self._spider.name+'-Scraper', 'IOError
 : '+
@@ -2756,59 +2756,25 @@
     
-print('%7B0%7D %5B%7B1%7D%5D Scraped: %7B2%7D'.format(fullstamp(), 
+self.logger.info(
 self
@@ -2792,13 +2792,24 @@
 me, 
+'Scraped: '+
 url)
-)
 %0A   
@@ -3038,16 +3038,16 @@
  = html%0A
+
         
@@ -3066,168 +3066,8 @@
 ts%0A%0A
-    def _debug(self, s):%0A        message = '%7B0%7D %5B%7B1%7D%5D %7B2%7D'.format(fullstamp(), self._spider.name, s)%0A        if self.args%5B'debug'%5D:%0A            print(message)%0A%0A
     

refactor scraper using logger