from typing import Any, Dict, List, Text

import regex
import re

from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
from rasa.nlu.training_data import Message


class WhitespaceTokenizer(Tokenizer):

    defaults = {
        # Flag to check whether to split intents
        "intent_tokenization_flag": False,
        # Symbol on which intent should be split
        "intent_split_symbol": "_",
    }

    def __init__(self, component_config: Dict[Text, Any] = None) -> None:
        """Construct a new tokenizer using the WhitespaceTokenizer framework."""

        super().__init__(component_config)

        self.emoji_pattern = self.get_emoji_regex()

    @staticmethod
    def get_emoji_regex():
        emoji_pattern = re.compile(
            "["
            "\U0001F600-\U0001F64F"  # emoticons
            "\U0001F300-\U0001F5FF"  # symbols & pictographs
            "\U0001F680-\U0001F6FF"  # transport & map symbols
            "\U0001F1E0-\U0001F1FF"  # flags (iOS)
            "\U00002702-\U000027B0"
            "\U000024C2-\U0001F251"
            "]+",
            flags=re.UNICODE,
        )
        return emoji_pattern

    def remove_emoji(self, text: Text) -> Text:

        return self.emoji_pattern.sub(r"", text)

    def tokenize(self, message: Message, attribute: Text) -> List[Token]:
        text = message.get(attribute)

        # we need to use regex instead of re, because of
        # https://stackoverflow.com/questions/12746458/python-unicode-regular-expression-matching-failing-with-some-unicode-characters

        # remove 'not a word character' if
        words = regex.sub(
            # there is a space or an end of a string after it
            r"[^\w#@&]+(?=\s|$)|"
            # there is a space or beginning of a string before it
            # not followed by a number
            r"(\s|^)[^\w#@&]+(?=[^0-9\s])|"
            # not in between numbers and not . or @ or & or - or #
            # e.g. 10'000.00 or blabla@gmail.com
            # and not url characters
            r"(?<=[^0-9\s])[^\w._~:/?#\[\]()@!$&*+,;=-]+(?=[^0-9\s])",
            " ",
            text,
        ).split()

        words = [self.remove_emoji(w) for w in words]
        words = [w for w in words if w]

        # if we removed everything like smiles `:)`, use the whole text as 1 token
        if not words:
            words = [text]

        return self._convert_words_to_tokens(words, text)

@@ -59,16 +59,58 @@
 ort re%0A%0A
+from constants import DOCS_URL_COMPONENTS%0A
 from ras
@@ -203,16 +203,57 @@
 Message%0A
+import rasa.utils.common as common_utils%0A
 %0A%0Aclass 
@@ -735,24 +735,291 @@
 ji_regex()%0A%0A
+        if %22case_sensitive%22 in self.component_config:%0A            common_utils.raise_warning(%0A                %22The option 'case_sensitive' was moved from the tokenizers to the %22%0A                %22featurizers.%22,%0A                docs=DOCS_URL_COMPONENTS,%0A            )%0A%0A
     @staticm

add warning