import re
from os import listdir, path
from html.parser import HTMLParser
from concurrent.futures import ThreadPoolExecutor
from pymongo import MongoClient

class MyHtmlParser(HTMLParser):

    def init(self):
        self.print = False
        self.div_count = 0
        self.div_open = False
        self.tr_open = False
        self.td_open = False
        self.profile_open = False
        self.single_advie = []

        self.doctor_advice = []
        self.profile = []

    def handle_data(self, data):
        if self.td_open:
            self.single_advie.append(data)
            self.td_open = False
        elif self.profile_open:
            self.profile.append(data)

    def handle_starttag(self, tag, attrs):
        if self.div_open:
            if tag == 'div':
                self.div_count += 1
            elif tag == 'tr':
                self.tr_open = True
            elif tag == 'td':
                self.td_open = True

        if tag == 'div' and self.check_style(attrs, "width: 724px; height: 332px; "):
            self.div_open = True
        elif tag == 'table' and self.check_style(attrs, "width: 1013px; "):
            self.profile_open = True

    def handle_endtag(self, tag):
        if self.profile_open and tag == 'table':
            self.profile_open = False

        elif  self.div_open:
            is_div = tag == "div"
            if is_div and self.div_count > 0:
                self.div_count -= 1
            elif is_div:
                self.div_open = False
            elif tag == 'tr':
                self.doctor_advice.append(self.single_advie)
                self.single_advie = []
                self.tr_open = False
            elif tag == 'td' and self.tr_open:
                if self.td_open:
                    self.single_advie.append('')
                    self.td_open = False

    def check_style(self, attrs, style):
        for attr in attrs:
            if attr[0] == 'style' and attr[1] == style:
                return True
        return False

def parse_one_file(file_path):
    parser = MyHtmlParser()
    parser.init()
    with open(file_path, encoding='UTF-8') as f:
        parser.feed(f.read())
    return parser.profile, parser.doctor_advice

def parse_one(dir, file_name):
    a, b = parse_one_file(dir + file_name)
    c, d = parse_one_file(dir + file_name.replace('.', 'S.'))
    b.extend(d)
    return {'_id':file_name.split('.')[0],'d':{'info':a, 'doctor_advice': b}}


def main():
    from datetime import datetime

    start = datetime.now()
    print('hello..')
    dir = "C:/data/xxxxxxxxx/data/"
    all_html = [f for f in listdir(dir) if f.endswith('htm')and len(f) == 10]
    client = MongoClient('192.168.2.110')
    collection = client.xinhuahos.paients
    for f in all_html:
        collection.save(parse_one(dir, f))
        print(f)
    client.close()

    print(datetime.now() - start)


if __name__ == '__main__':
    main()

@@ -371,32 +371,112 @@
 le_open = False%0A
+        self.profile_keyword = '%E8%AF%A6%E7%BB%86%E5%86%85%E5%AE%B9'%0A        self.profile_keyword_open = False%0A
         self.sin
@@ -600,32 +600,181 @@
 f self.td_open:%0A
+            if self.profile_open:%0A                self.profile.append(data)%0A                self.td_open = False%0A            elif self.div_open:%0A    
             self
@@ -804,32 +804,36 @@
 ta)%0A            
+ 
+   
 self.td_open = F
@@ -829,32 +829,33 @@
 td_open = False%0A
+%0A
         elif sel
@@ -907,16 +907,102 @@
 nd(data)
+%0A        if data == self.profile_keyword:%0A            self.profile_keyword_open = True
 %0A%0A    de
@@ -1258,24 +1258,120 @@
 pen = True%0A%0A
+        elif self.profile_open:%0A            if tag == 'td':%0A                self.td_open = True%0A
         if t
@@ -1505,24 +1505,54 @@
 'table' and 
+(self.profile_keyword_open or 
 self.check_s
@@ -1577,24 +1577,25 @@
 : 1013px; %22)
+)
 :%0A          
@@ -1734,32 +1734,78 @@
 ile_open = False
+%0A            self.profile_keyword_open = False
 %0A%0A        elif  
@@ -2331,24 +2331,186 @@
 pen = False%0A
+            elif tag == 'td' and self.profile_open and self.td_open:%0A                self.profile.append('')%0A                self.td_open = False%0A                
 %0A    def che
@@ -3402,16 +3402,121 @@
 paients%0A
+    # collection.save(parse_one(dir, 'B17493.htm'))%0A    # collection.save(parse_one(dir, 'B09560.htm'))%0A%0A
     for 

add new keyword to check if the info started