import re
import json


def fix_tokens_file(file_path):
    """
    Replace each

    # ::tok sentence

    by json parsable version

    # ::token json-parseable-sentence

    so that

    sentence == json.loads(json-parseable-sentence)
    """

    token_line = re.compile('^# ::tok (.*)')

    # read and modifiy token lines
    new_amr = []
    tokens = []
    with open(file_path) as fid:
        for line in fid:
            fetch = token_line.match(line.rstrip())
            if fetch:
                sentence = fetch.groups()[0]
                tokens.append(sentence)
                json_str = json.dumps(sentence)
                new_amr.append(f'# ::tokens {json_str}\n')
            else:
                new_amr.append(line)

    return new_amr, tokens