import re


def basic_tokenizer(line: str, normalize_digits=True):
    """ Basic tokenizer for raw text. """

    WORD_SPLIT = re.compile("([.,!?\"'-<>:;)(])")
    DIGIT_RE = re.compile(r"\d")

    # Clean sentence.
    line = re.sub('<u>', '', line)
    line = re.sub('</u>', '', line)
    line = re.sub('\[', '', line)
    line = re.sub('\]', '', line)

    words = []
    # For each word:
    for fragment in line.strip().lower().split():
        # For each token:
        for token in re.split(WORD_SPLIT, fragment):
            # If it an empty list - not token:
            if not token:
                continue
            # If normalize digits:
            if normalize_digits:
                # Replace digits by token #.
                token = re.sub(DIGIT_RE, '#', token)
            # Store token.
            words.append(token)
    return words