@@ -0,0 +1,1332 @@
+'''%0AA toolkit for verifying the ip address of search engine spiders.%0A'''%0Aimport socket%0A%0AHOST_NAME_MAP = %7B%0A    'baidu': '.crawl.baidu.com',%0A    'sogou': '.crawl.sogou.com',%0A    'google': '.googlebot.com',%0A    'bing': '.search.msn.com',%0A    'yandex': '.yandex.com',%0A    'ahrefs': '.ahrefs.com'%0A%7D%0A%0AUA_MAP = %7B%0A    'baidu': 'baiduspider',%0A    'sogou': 'sogou web spider',%0A    'google': 'googlebot',%0A    'bing': 'bingbot',%0A    'yandex': 'yandexbot',%0A    'ahrefs': 'ahrefsbot',%0A    'shenma': 'yisouspider',%0A    '360': '360spider'%0A%7D%0A%0A%0Adef get_verified_spider_name(ip, spider_name=None):%0A    '''%0A    Return verified spider name based on ip address.%0A    '''%0A    hostnames = %5B%5D%0A    try:%0A        (hostname, aliaslist, __) = socket.gethostbyaddr(ip)%0A        hostnames.append(hostname)%0A        hostnames.extend(aliaslist)%0A    except Exception as e:%0A        pass%0A    if not hostnames:%0A        return None%0A    for k, v in HOST_NAME_MAP.items():%0A        if spider_name and spider_name != k:%0A            continue%0A        for hostname in hostnames:%0A            if hostname.endswith(v):%0A                return k%0A    return None%0A%0A%0Adef guess_spider_name_from_ua(ua):%0A    '''%0A    Guess spider name from browser user agent string.%0A    '''%0A    ua = (ua or '').lower()%0A    for k, v in UA_MAP.items():%0A        if v in ua:%0A            return k%0A    return None%0A

add sitemap.spider module for verifying spider via ip or ua