import re
import string
from sets import Set

import nltk
from nltk.tokenize import sent_tokenize
from nltk.tokenize import word_tokenize
from wedc.domain.core.common import str_helper
from wedc.domain.vendor.nltk import stopword
from wedc.domain.vendor.nltk import stem
from wedc.domain.core.http import domain
from wedc.domain.core.common import stopword_helper




trantab = string.maketrans(string.punctuation,' '*(len(string.punctuation)))

class DataParser():
    def __init__(self):
        self.stopset = self.load_stopset()

    def load_stopset(self):
        stops = stopword.get_stopwords()
        names = stopword_helper.get_person_names()
        # country, country_abbr = stopword_helper.get_country_names()
        nationality = stopword_helper.get_nationality_names()
        stop_set = Set(stops) | Set(names) | Set(nationality) # | Set(country) # | Set(country_abbr) 
        stop_set = [str(stem.stemming(_).strip()) for _ in stop_set]
        return stop_set

    def parse(self, text):
        text = self.text_preprocessing(text)
        
        sentences = [sentence for sentence in sent_tokenize(text)]
        # sentences = [nltk.word_tokenize(sent) for sent in sentences]


        print sentences
        # self.sentence_operation(sentence)
        # sentences = self.sentence_operation(' '.join(sentences).lower())

        # tokens = [self.token_operation(token) for token in word_tokenize(sentences) if token not in stop and not has_url(token)]
        # tokens = [_ for _ in tokens if _ and _ != ' ' and _ not in stop_set]
        # ans = ' '.join(list(set(tokens)))
        # return str(ans)
        # 

    def text_preprocessing(self, text):

        text = text.encode('ascii', 'ignore')

        # remove tags
        text = re.sub(r'<[^>]+>', ' ', text)

        # remove url
        text = re.sub(r'\w+:\/{2}[\d\w-]+(\.[\d\w-]+)*(?:(?:\/[^\s/]*))*', '', text)
        text = re.sub(r'^[a-z0-9\-\.]+\.(com|org|net|me|cn|biz|us)$', '', text.lower())

        # remove space
        text = re.sub(r'([\t\n\r]|\\+)', ' ', text)

        # text = text.translate(trantab)
        return ' '.join(text.split())  
    



"""
# for more domain ext
# domain_ext_list = domain.get_domain_ext_list()
# text = re.sub(r'^[a-z0-9\-\.]+\.('+'|'.join(domain_ext_list)+')$', '', text)

"""
@@ -1046,24 +1046,16 @@
 ng(text)
-%0A       
  %0A      
@@ -1060,139 +1060,93 @@
     
-sentences = %5Bsentence for sentence in sent_tokenize(text)%5D%0A        # sentences = %5Bnltk.word_tokenize(sent) for sent in sentences%5D%0A%0A
+tokens = %5Bself.token_preprocessing(token) for token in word_tokenize(text)%5D%0A%0A        
 %0A   
@@ -1156,25 +1156,25 @@
   print 
-sentences
+text%0A    
 %0A       
@@ -1565,27 +1565,16 @@
 str(ans)
-%0A        # 
 %0A%0A    de
@@ -1654,16 +1654,45 @@
 gnore')%0A
+        text = unescape(text)
 %0A       
@@ -2015,32 +2015,122 @@
 )', ' ', text)%0A%0A
+        # remove space%0A        text = re.sub(r'%5B' + string.punctuation +'%5D+', ' ', text)%0A%0A
         # text =
@@ -2198,14 +2198,1053 @@
 )  %0A
+%0A
     
-%0A%0A
+def token_preprocessing(self, token):%0A        # %0A        if re.search(r'(%5Cd+%5Bk$%5D+%5B/(hr%7Chour)%5D*%7C401%5B%5Cw%5Cd%5D*)', token):%0A            return '401k'%0A        if re.search(r'%5Cd', token):%0A            return None%0A        # if len(token) == 1:%0A        #     return None%0A        %0A%0A%0A        token = stem.stemming(token).strip()%0A        if token in self.stopset:%0A            return None%0A%0A        return %0A%0A%0Adef unescape(text):%0A    import re, htmlentitydefs%0A    def fixup(m):%0A        text = m.group(0)%0A        if text%5B:2%5D == %22&#%22:%0A            # character reference%0A            try:%0A                if text%5B:3%5D == %22&#x%22:%0A                    return unichr(int(text%5B3:-1%5D, 16))%0A                else:%0A                    return unichr(int(text%5B2:-1%5D))%0A            except ValueError:%0A                pass%0A        else:%0A            # named entity%0A            try:%0A                text = unichr(htmlentitydefs.name2codepoint%5Btext%5B1:-1%5D%5D)%0A            except KeyError:%0A                pass%0A        return text # leave as is%0A    return re.sub(%22&#?%5Cw+;%22, fixup, text)
 %0A%0A%22%22
@@ -3388,19 +3388,170 @@
 )$', '', text)%0A%0A
+# for sentence processing%0A# sentences = %5Bsentence for sentence in sent_tokenize(text)%5D%0A# sentences = %5Bnltk.word_tokenize(sent) for sent in sentences%5D%0A%0A
 %22%22%22

refine text preprocessing