import re


def tokenize_soft(caption, lowercase=True):
    """
    Tokenization used for the icann paper:
        * Removes very little punctuation
        * Lowercase
    :param caption: String to tokenize
    :param lowercase: Whether to lowercase the caption or not
    :return: Tokenized version of caption
    """
    tokenized = re.sub(u'[\n\t]+', u'', caption.strip())
    tokenized = re.sub(u'[\.]+', u' . ', tokenized)
    tokenized = re.sub(u'[,]+', u' , ', tokenized)
    tokenized = re.sub(u'[!]+', u' ! ', tokenized)
    tokenized = re.sub(u'[?]+', u' ? ', tokenized)
    tokenized = re.sub(u'[\{]+', u' { ', tokenized)
    tokenized = re.sub(u'[\}]+', u' } ', tokenized)
    tokenized = re.sub(u'[\(]+', u' ( ', tokenized)
    tokenized = re.sub(u'[\)]+', u' ) ', tokenized)
    tokenized = re.sub(u'[\[]+', u' [ ', tokenized)
    tokenized = re.sub(u'[\]]+', u' ] ', tokenized)
    tokenized = re.sub(u'["]+', u' " ', tokenized)
    tokenized = re.sub(u'[\']+', u" ' ", tokenized)
    tokenized = re.sub(u'[ ]+', u' ', tokenized)
    # tokenized = u" ".join(tokenized)
    return tokenized if not lowercase else tokenized.lower()