# -*- coding: utf-8 -*-
from nltk.chunk import *
from nltk.chunk.util import *
from nltk.chunk.regexp import *
from nltk import Tree
from nltk.tokenize import word_tokenize
import nltk

import util


def filter_paren(tup):
    if tup[0] == '(':
        return tup[0], '-NONE-'
    return tup


def parse(text):
    tokens = word_tokenize(util.preproces_text(text))
    tagged = nltk.pos_tag(tokens)

    # get rid of commas
    tagged = [tup for tup in tagged if tup[0] != ',']

    # flag open paren as -NONE-
    tagged = map(filter_paren, tagged)

    grammer = 'Location: ' \
        '{<CD><NNP>+<JJ>?<NNP>+|' \
        '<CD><NNP><CD><NNP>+|' \
        '<CD>+<NNP>+|' \
        '<CD><NNP>+}'
    chunkParser = nltk.RegexpParser(grammer)
    result = chunkParser.parse(tagged)
    return [s for s in result.subtrees(lambda t: t.label() == 'Location')]

if __name__ == '__main__':
    import codecs

    sample = codecs.open('trainers/ad-trainer2.txt', 'r', encoding='utf8') \
        .read()

    util.showResults(parse(sample))

@@ -191,16 +191,33 @@
 rt util%0A
+import usaddress%0A
 %0A%0Adef fi
@@ -309,20 +309,32 @@
 %0A%0A%0Adef p
-a
 r
+obableAddres
 se
+s
 (text):%0A
@@ -878,16 +878,395 @@
 ion')%5D%0A%0A
+def isValidAddress(ady):%0A    address = usaddress.parse(ady)%0A    if len(address) %3C 5:%0A        return False%0A%0A    if any(%5Ba%5B1%5D == 'Recipient' for a in address%5D):%0A        return False%0A%0A    return True%0A%0Adef parse(text):%0A    candidates = probableAddresses(text)%0A    candidates = %5Butil.location_to_string(c) for c in candidates%5D%0A    return %5Bc for c in candidates if isValidAddress(c)%5D%0A%0A
 if __nam
@@ -1352,9 +1352,9 @@
 iner
-2
+1
 .txt
@@ -1405,25 +1405,14 @@
     
-util.showResults(
+print 
 pars
@@ -1420,10 +1420,9 @@
 (sample)
-)
 %0A

implement validator based on usaddress