import re


def char_tokenizer(sentence):
    """Cut the sentence into the format we want:
    - continuous letters and symbols like back-slash and parenthese
    - single Chinese character
    - other symbols
    """
    regex = []
    # English and number part for type name.
    regex += [r'[0-9a-zA-Z\\+()\-<>]+']
    # Chinese characters part.
    regex += [r'[\u4e00-\ufaff]']
    # Exclude the space.
    regex += [r'[^\s]']
    regex = '|'.join(regex)
    _RE = re.compile(regex)
    segs = _RE.findall(sentence.strip())
    return segs