def get_text_inputs(tokens, mask_token,
                    bpe_whitespace_tok="▁",
                    single_mask_tok_per_masked_word=True,
                    eos_symbol="</s>"):
    """
    tokens:
    [
        {
            'word': "▁",
            'word_pos': 1,
            'mask': False
        },
        {
            'word': "hello",
            'word_pos': 2,
            'mask': True
        },
        {
            'word': "▁",
            'word_pos': 1,
            'mask': False
        },
        {
            'word': "world",
            'word_pos': 3,
            'mask': False
        },
        {
            'word': "</s>",
            'word_pos': 4,
            'mask': False
        },
    ]
    
    ->
    
    graphemes = [▁, MASK, MASK, MASK, MASK, MASK, ▁, w, o, r, l, d, </s>]
    word_pos_of_graphemes = [1 2 2 2 2 2 1 3 3 3 3 3 4]
    """
    graphemes = []
    word_pos_of_graphemes = []

    for token in tokens:
        if token["word"] in [bpe_whitespace_tok, eos_symbol]:
            graphemes.append(token["word"])
            word_pos_of_graphemes.append(token["word_pos"])
        else:
            if token["mask"]: # masked word
                for c in token["word"]:
                    graphemes.append(mask_token)
                    word_pos_of_graphemes.append(token["word_pos"])
                    if single_mask_tok_per_masked_word:
                        break
            else: # unmasked word
                for c in token["word"]:
                    graphemes.append(c)
                    word_pos_of_graphemes.append(token["word_pos"])

    return graphemes, word_pos_of_graphemes