from typing import OrderedDict


def __input_vocab(nlp, datasets):
    """
    Extracts the set of natural language tokens from
    each source sample in each dataset split and builds
    the encoder vocabulary.

    :param nlp:         nl processing and parsing utils.
    :param datasets:    all dataset splits with source and
                        target samples.
    :returns:           'i2w' and 'w2i' dictionaries taking
                        indices to tokens and vice versa.
    """

    vocab = OrderedDict()

    # Meta-Symbols.
    marks = nlp.mark.inp.values()
    marks = {m: None for m in marks}
    vocab.update(marks)

    for dataset_name in datasets:
        src = datasets[dataset_name]['src']

        for sample in src:
            tokens = nlp.normalize(sample)
            tokens = {t: None for t in tokens}
            vocab.update(tokens)

    vocab = [t for t in vocab]
    i2w = {i: t for i, t in enumerate(vocab)}
    w2i = {t: i for i, t in enumerate(vocab)}
    return i2w, w2i