from collections import defaultdict
from sys import stdout
from preprocess.data import process_raw
from word2vec import WORD_EMBEDDING_BIN_FILE
from random import sample
import pickle as pkl
import re

FILE = './data/paraphrases.wikianswer.txt'

if __name__ == '__main__':
    # generate hash map
    # paraphrase questions mapping
    para_lemma_map = defaultdict(list)
    i = 0
    stdout.write("Loading wikianswers-paraphrases-1.0/word_alignments.txt\n")
    with open('./data/wikianswers-paraphrases-1.0/word_alignments.txt', 'r') as f:
        for line in f:
            content = line.strip()
            q1, q2, _ = content.split('\t')
            para_lemma_map[q1].append(q2)
            i += 1
            stdout.write("\rloaded: %.2f%%" % (float(i) / 35291309 * 100))
            stdout.flush()
    stdout.write("\n")

    # origin sentence - lemma mapping
    sent_lemma_map = {}
    i = 0
    stdout.write("Loading wikianswers-paraphrases-1.0/questions.txt\n")
    with open('./data/wikianswers-paraphrases-1.0/questions.txt', 'r') as f:
        for line in f:
            content = line.strip()
            try:
                # question, tokens, POS, lemma
                _, q, _, lemma = content.split('\t')
                sent_lemma_map[lemma] = q
                i += 1
                stdout.write("\rloaded: %.2f%%" % (float(i) / 2585750 * 100))
                stdout.flush()
            except ValueError:
                continue
    stdout.write("\n")

    stdout.write("Loading word embedding hashmap\n")
    with open(WORD_EMBEDDING_BIN_FILE, 'rb') as f:
        emb_voc_dict = pkl.load(f)
    word_embedding_keys = emb_voc_dict.keys()

    i = 0
    stdout.write("Generating paraphrases.wikianswer.txt\n")
    with open(FILE, 'a') as fw:
        with open('./data/wikianswers-paraphrases-1.0/questions.txt', 'r') as fr:
            for line in fr:
                if i > 300000:
                    break

                content = line.strip()
                try:
                    # question, tokens, POS, lemma
                    _, q, _, lemma = content.split('\t')
                    q = q.lower()
                    # remove symbols
                    q_tokens = re.findall('[a-z0-9]+', q)
                    # use tokens to regenerate sentence
                    q = " ".join(q_tokens) + " ?"

                    # make sure all tokens have the embedding
                    if any([token not in word_embedding_keys for token in q_tokens]):
                        continue

                    # get corresponde lemma sentences
                    q_para_lemma_list = para_lemma_map[lemma]

                    # generate paraphrase pairs
                    for q in q_para_lemma_list:
                        # for each lemma sentence, find its original sentence
                        q_para = sent_lemma_map[q_para_lemma]

                        q_para_lemma = q_para.lower()
                        # remove symbols
                        q_para_lemma_tokens = re.findall('[a-z0-9]+', q_para_lemma)
                        # use tokens to regenerate sentence
                        q_para = " ".join(q_para_lemma_tokens) + " ?"

                        # generalize the sentence, remember to add ? in the end
                        # record down
                        fw.write("{}\t{}\n".format(q, q_para))
                        i += 1
                        stdout.write("\rgenerated: %d" % i)
                        stdout.flush()
                except ValueError:
                    continue
                except KeyError:
                    continue
    stdout.write("\nTotal: %d\n" % i)

@@ -1722,32 +1722,83 @@
 ianswer.txt%5Cn%22)%0A
+    valid_start = %5B%22who%22, %22what%22, %22when%22, %22where%22%5D%0A
     with open(FI
@@ -1813,16 +1813,16 @@
  as fw:%0A
-
         
@@ -2278,16 +2278,17 @@
 %5D+', q)%0A
+%0A
         
@@ -2376,32 +2376,198 @@
 q_tokens) + %22 ?%22
+%0A                    # sentence start with %22who%22, %22what%22 ... %0A                    if not any(%5Bq.startswith(p) for p in valid_start%5D):%0A                        continue
 %0A%0A              
@@ -2924,16 +2924,27 @@
    for q
+_para_lemma
  in q_pa
@@ -3330,32 +3330,32 @@
 nerate sentence%0A
-
                 
@@ -3407,16 +3407,199 @@
 ) + %22 ?%22
+%0A                        # sentence start with %22who%22, %22what%22 ... %0A                        if not any(%5Bq_para.startswith(p) for p in valid_start%5D):%0A                            continue
 %0A%0A      

ignore sentence not start with regular question start words