import logging


def vectorize_all(examples, word_dict, char_dict, ner_dict,
              sort_by_len=True, verbose=True, concat=False):
    """
        examples:documents, questions, options, answers, doc_char, ques_char, opt_char, 
                            doc_ners, ques_ners, opt_ners, document_co_occur_features
        Vectorize all features including word embedding, char embedding and ner embedding
        in_x1: word embeddings for doc
        in_x2: word embeddings for ques
        in_x3: word embeddings for options
        in_x4: char embeddings for doc
        in_x5: char embeddings for ques
        in_x6: char embeddings for options
        in_x7: ner embeddings for doc
        in_x8: ner embeddings for ques
        in_x9: ner embeddings for options
        in_x10: co-features for doc
        in_y: label
        in_l: whether the entity label occurs in the document.
    """
    in_x1 = []
    in_x2 = []
    in_x3 = []
    in_x4 = []
    in_x5 = []
    in_x6 = []
    in_x7 = []
    in_x8 = []
    in_x9 = []
    in_x10 = []
    in_y = []
    def get_vector(st, dic):
        seq = [dic[w] if w in dic else 1 for w in st]
        return seq

    def get_vector_char(st, dic):
        seq = [[dic[c] if c in dic else 1 for c in w] for w in st]
        return seq
    op_w = examples[2]
    op_c = examples[6]
    op_ner = examples[9]
    examples = list(examples[0:2]) + [examples[3]] + list(examples[4:6]) + list(examples[7:9]) + [examples[10]]
    for idx, (d, q, a, d_c, q_c, d_ner, q_ner, d_co_f) in enumerate(zip(*examples)):
        d_words = d.split(' ')
        q_words = q.split(' ')
        d_char = d_c.split(' ')
        q_char = q_c.split(' ')
        d_ner = d_ner.split(' ')
        q_ner = q_ner.split(' ')
        # assert 0 <= a <= 3
        seq1 = get_vector(d_words, word_dict)
        seq2 = get_vector(q_words, word_dict)
        seq3 = get_vector_char(d_char, char_dict)
        seq4 = get_vector_char(q_char, char_dict)
        seq5 = get_vector(d_ner, ner_dict)
        seq6 = get_vector(q_ner, ner_dict)
        if (len(seq1) > 0) and (len(seq2) > 0):
            in_x1 += [seq1]
            in_x2 += [seq2]
            in_x4 += [seq3]
            in_x5 += [seq4]
            in_x7 += [seq5]
            in_x8 += [seq6]
            in_x10 += [d_co_f]
            option_seq = []
            option_char_seq = []
            option_ner_seq = []
            for i in range(4):
                if concat:
                    op = " ".join(q_words) + ' @ ' + op_w[i + idx * 4]
                    op_char_one = q_c + ' @ ' + op_c[i + idx * 4]
                    op_ner_one = " ".join(q_ner) + ' @ ' + op_ner[i + idx * 4]
                else:
                    op = op_w[i + idx * 4]
                    op_char_one = op_c[i + idx * 4]
                    op_ner_one = op_ner[i + idx * 4]
                op = op.split(' ')
                option = get_vector(op, word_dict)
                op_char_one = op_char_one.split(' ')
                op_char_one = get_vector_char(op_char_one, char_dict)
                op_ner_one = op_ner_one.split(' ')
                op_ner_one = get_vector(op_ner_one, ner_dict)
                assert len(option) > 0
                option_seq += [option]
                option_char_seq += [op_char_one]
                option_ner_seq += [op_ner_one]

            in_x3 += [option_seq]
            in_x6 += [option_char_seq]
            in_x9 += [option_ner_seq]
            in_y.append(a)
        if verbose and (idx % 10000 == 0):
            logging.info('Vectorization: processed %d / %d' % (idx, len(examples[0])))

    def len_argsort(seq):
        return sorted(range(len(seq)), key=lambda x: len(seq[x]))

    if sort_by_len:
        # sort by the document length
        sorted_index = len_argsort(in_x1)
        in_x1 = [in_x1[i] for i in sorted_index]
        in_x2 = [in_x2[i] for i in sorted_index]
        in_x3 = [in_x3[i] for i in sorted_index]
        in_x4 = [in_x4[i] for i in sorted_index]
        in_x5 = [in_x5[i] for i in sorted_index]
        in_x6 = [in_x6[i] for i in sorted_index]
        in_x7 = [in_x7[i] for i in sorted_index]
        in_x8 = [in_x8[i] for i in sorted_index]
        in_x9 = [in_x9[i] for i in sorted_index]
        in_x10 = [in_x10[i] for i in sorted_index]
        in_y = [in_y[i] for i in sorted_index]
    new_in_x3 = []
    new_in_x6 = []
    new_in_x9 = []
    for i in in_x3:
        new_in_x3 += i
    for i in in_x6:
        new_in_x6 += i
    for i in in_x9:
        new_in_x9 += i
    return in_x1, in_x2, new_in_x3, in_x4, in_x5, new_in_x6, in_x7, in_x8, new_in_x9, in_x10, in_y