import re


def tokenize_basic(caption, lowercase=True):
    """
    Basic tokenizer for the input/output data of type 'text':
       * Splits punctuation
       * Optional lowercasing

    :param caption: String to tokenize
    :param lowercase: Whether to lowercase the caption or not
    :return: Tokenized version of caption
    """

    punct = [u'.',
             u';',
             u',',
             u'，',
             u"/",
             u'[',
             u']',
             u'"',
             u'{',
             u'}',
             u'(',
             u')',
             u'=',
             u'+',
             u'\\',
             u'_',
             u'-',
             u'>',
             u'<',
             u'@',
             u'`',
             u'¿',
             u'?',
             u'¡',
             u'!']

    def processPunctuation(inText):
        """
        Removes punctuation.
        """
        outText = inText
        for p in punct:
            outText = outText.replace(p, u' ' + p + u' ')
        return outText

    resAns = caption.lower() if lowercase else caption
    resAns = resAns.replace(u'\n', u' ')
    resAns = resAns.replace(u'\t', u' ')
    resAns = processPunctuation(resAns)
    resAns = re.sub(u'[ ]+', u' ', resAns)
    return resAns