# 解析器
import types
from bs4 import BeautifulSoup
from urllib.parse import urlparse


class Parser:
    def __init__(self, func_parse_url=None, func_parse_content=None):
        self.html = None
        self.soup = None
        self.content = None
        self.urls = []
        self.base_url = None
        self.filter_str_list = []

        # 自定义解析器
        if func_parse_url is not None:
            self.parse_url = types.MethodType(func_parse_url, self)

        if func_parse_content is not None:
            self.parse_content = types.MethodType(func_parse_content, self)

    # 解析内容
    def parse_content(self):
        self.content = self.soup.body

    # 解析 url
    def parse_url(self):
        node = self.soup.find_all('a')
        for n in node:
            url = self.filter_url(n.get('href', ''))
            if url:
                self.urls.append(url)

    # 过滤url
    def filter_url(self, url):
        parse_url = urlparse(url)
        if not url:
            return False
        elif url == '/' or url == '#' or 'javascript:' in url:
            return False
        # 判断 scheme
        elif parse_url.scheme and parse_url.scheme != self.base_url.scheme:
            return False
        # 判断 url 是否属于当前域名
        elif parse_url.netloc and parse_url.netloc not in self.base_url.netloc:
            return False

        if self.filter_str_list:
            for str_ in self.filter_str_list:
                if str_ in url:
                    return False

        if not parse_url.netloc:
            url = 'http://' + self.base_url.netloc + url
        return url

    def set_html(self, html):
        self.html = html
        self.soup = BeautifulSoup(self.html, 'html.parser', from_encoding='utf8')

    def set_base_url(self, base_url):
        self.base_url = urlparse(base_url)

    def add_filter_str(self, filter_str):
        self.filter_str_list.append(filter_str)

    def get_content(self):
        return self.content

    def get_url(self):
        return self.urls

'''
html_ = """
<html><head><title>The Dormouse's story</title></head>
<body>
<p class="title" name="dromouse"><b>The Dormouse's story</b></p>
<p class="story">Once upon a time there were three little sisters; and their names were
<a href="http://example.com/elsie" class="sister" id="link1"><!-- Elsie --></a>,
<a href="http://example.com/lacie" class="sister" id="link2">Lacie</a> and
<a href="http://example.com/tillie" class="sister" id="link3">Tillie</a>;
and they lived at the bottom of a well.</p>
<p class="story">...</p>
</body>
"""
a = Parser()
a.set_html(html_)
a.set_base_url('http://example.com/')
a.parse_url()
a.parse_content()
print(a.get_url())
'''

@@ -320,17 +320,100 @@
 str_
-lis
+dic
 t = 
-%5B%5D
+%7B%0A            'equal': %5B'/', '#', 'javascript:'%5D,%0A            'contain': %5B%5D%0A        %7D
 %0A%0A  
@@ -423,16 +423,19 @@
    # %E8%87%AA%E5%AE%9A%E4%B9%89
+url
 %E8%A7%A3%E6%9E%90%E5%99%A8%0A    
@@ -533,24 +533,42 @@
 _url, self)%0A
+        # %E8%87%AA%E5%AE%9A%E4%B9%89%E5%86%85%E5%AE%B9%E8%A7%A3%E6%9E%90%E5%99%A8
 %0A        if 
@@ -877,24 +877,8 @@
 l = 
-self.filter_url(
 n.ge
@@ -890,17 +890,16 @@
 ef', '')
-)
 %0A       
@@ -931,51 +931,19 @@
     
-self.urls.append(url)%0A%0A    # %E8%BF%87%E6%BB%A4url%0A    def 
+url = self.
 filt
@@ -953,19 +953,12 @@
 url(
-self, 
 url)
-:
 %0A   
@@ -966,49 +966,19 @@
     
-parse_url = urlparse(url)%0A
+ 
        
- 
 if 
-not 
 url:
@@ -990,37 +990,24 @@
         
-return False%0A
         
 elif url
@@ -1002,61 +1002,81 @@
     
+s
 el
-if url == '/' or url == '#' or 'javascript:' in
+f.urls.append(url)%0A%0A    # %E8%BF%87%E6%BB%A4url TODO %E5%85%BC%E5%AE%B9%E6%80%A7%0A    def filter_url(self,
  url
+)
 :%0A  
@@ -1081,32 +1081,41 @@
         
-    return False
+parse_url = urlparse(url)
 %0A       
@@ -1127,34 +1127,32 @@
  scheme%0A        
-el
 if parse_url.sch
@@ -1377,36 +1377,210 @@
 self.filter_str_
-list
+dict%5B'equal'%5D:%0A            for str_ in self.filter_str_dict%5B'equal'%5D:%0A                if str_ == url:%0A                    return False%0A%0A        if self.filter_str_dict%5B'contain'%5D
 :%0A            fo
@@ -1605,20 +1605,31 @@
 ter_str_
-list
+dict%5B'contain'%5D
 :%0A      
@@ -2088,31 +2088,109 @@
 str_
-list.append(filter_str)
+dict%5B'equal'%5D += filter_str%5B'equal'%5D%0A        self.filter_str_dict%5B'contain'%5D += filter_str%5B'contain'%5D
 %0A%0A  

optimize code