'''simple docstring'''





import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class   UpperCamelCase					(       a_    ,     unittest.TestCase       ):







    """simple docstring"""





    A     :    Optional[int]				=			RealmTokenizer
    A     :    Optional[Any]				=			RealmTokenizerFast
    A     :    Optional[int]				=			True
    A     :    Dict				=			True
    A     :    Union[str, Any]				=			filter_non_english





    def    SCREAMING_SNAKE_CASE_     (					self     :						Optional[int]):


         """simple docstring"""







         super().setUp()

         a					:							Optional[Any]							= [
             '[UNK]',
             '[CLS]',
             '[SEP]',
             '[PAD]',
             '[MASK]',
             'want',
             '##want',
             '##ed',
             'wa',
             'un',
             'runn',
             '##ing',
             ',',
             'low',
             'lowest',
         ]
         a					:							Tuple							= os.path.join(self.tmpdirname							,							VOCAB_FILES_NAMES['vocab_file'])
         with open(self.vocab_file							,							'w'							,							encoding='utf-8') as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))





    def    SCREAMING_SNAKE_CASE_     (					self     :						Dict							,							UpperCAmelCase_     :						str):


         """simple docstring"""







         a					:							int							= 'UNwant\u00E9d,running'
         a					:							List[str]							= 'unwanted, running'
         return input_text, output_text





    def    SCREAMING_SNAKE_CASE_     (					self     :						Any):


         """simple docstring"""







         a					:							Optional[int]							= self.tokenizer_class(self.vocab_file)

         a					:							Tuple							= tokenizer.tokenize('UNwant\u00E9d,running')
         self.assertListEqual(UpperCAmelCase_							,							['un', '##want', '##ed', ',', 'runn', '##ing'])
         self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_)							,							[9, 6, 7, 1_2, 1_0, 1_1])





    def    SCREAMING_SNAKE_CASE_     (					self     :						str):


         """simple docstring"""







         if not self.test_rust_tokenizer:
              return

         a					:							Optional[int]							= self.get_tokenizer()
         a					:							List[str]							= self.get_rust_tokenizer()

         a					:							int							= 'UNwant\u00E9d,running'

         a					:							str							= tokenizer.tokenize(UpperCAmelCase_)
         a					:							List[str]							= rust_tokenizer.tokenize(UpperCAmelCase_)
         self.assertListEqual(UpperCAmelCase_							,							UpperCAmelCase_)

         a					:							Tuple							= tokenizer.encode(UpperCAmelCase_							,							add_special_tokens=UpperCAmelCase_)
         a					:							List[Any]							= rust_tokenizer.encode(UpperCAmelCase_							,							add_special_tokens=UpperCAmelCase_)
         self.assertListEqual(UpperCAmelCase_							,							UpperCAmelCase_)

         a					:							int							= self.get_rust_tokenizer()
         a					:							List[str]							= tokenizer.encode(UpperCAmelCase_)
         a					:							int							= rust_tokenizer.encode(UpperCAmelCase_)
         self.assertListEqual(UpperCAmelCase_							,							UpperCAmelCase_)

         # With lower casing
         a					:							List[str]							= self.get_tokenizer(do_lower_case=UpperCAmelCase_)
         a					:							Optional[int]							= self.get_rust_tokenizer(do_lower_case=UpperCAmelCase_)

         a					:							Optional[Any]							= 'UNwant\u00E9d,running'

         a					:							List[Any]							= tokenizer.tokenize(UpperCAmelCase_)
         a					:							List[str]							= rust_tokenizer.tokenize(UpperCAmelCase_)
         self.assertListEqual(UpperCAmelCase_							,							UpperCAmelCase_)

         a					:							List[Any]							= tokenizer.encode(UpperCAmelCase_							,							add_special_tokens=UpperCAmelCase_)
         a					:							Any							= rust_tokenizer.encode(UpperCAmelCase_							,							add_special_tokens=UpperCAmelCase_)
         self.assertListEqual(UpperCAmelCase_							,							UpperCAmelCase_)

         a					:							Dict							= self.get_rust_tokenizer()
         a					:							Optional[Any]							= tokenizer.encode(UpperCAmelCase_)
         a					:							Optional[int]							= rust_tokenizer.encode(UpperCAmelCase_)
         self.assertListEqual(UpperCAmelCase_							,							UpperCAmelCase_)





    def    SCREAMING_SNAKE_CASE_     (					self     :						Tuple):


         """simple docstring"""







         a					:							Any							= BasicTokenizer()

         self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz')							,							['ah', '\u535A', '\u63A8', 'zz'])





    def    SCREAMING_SNAKE_CASE_     (					self     :						Union[str, Any]):


         """simple docstring"""







         a					:							int							= BasicTokenizer(do_lower_case=UpperCAmelCase_)

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')							,							['hello', '!', 'how', 'are', 'you', '?'])
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo')							,							['hello'])





    def    SCREAMING_SNAKE_CASE_     (					self     :						Tuple):


         """simple docstring"""







         a					:							Dict							= BasicTokenizer(do_lower_case=UpperCAmelCase_							,							strip_accents=UpperCAmelCase_)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')							,							['hällo', '!', 'how', 'are', 'you', '?'])
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo')							,							['h\u00E9llo'])





    def    SCREAMING_SNAKE_CASE_     (					self     :						List[Any]):


         """simple docstring"""







         a					:							str							= BasicTokenizer(do_lower_case=UpperCAmelCase_							,							strip_accents=UpperCAmelCase_)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')							,							['hallo', '!', 'how', 'are', 'you', '?'])
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo')							,							['hello'])





    def    SCREAMING_SNAKE_CASE_     (					self     :						Any):


         """simple docstring"""







         a					:							Optional[Any]							= BasicTokenizer(do_lower_case=UpperCAmelCase_)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')							,							['hallo', '!', 'how', 'are', 'you', '?'])
         self.assertListEqual(tokenizer.tokenize('H\u00E9llo')							,							['hello'])





    def    SCREAMING_SNAKE_CASE_     (					self     :						List[Any]):


         """simple docstring"""







         a					:							Optional[Any]							= BasicTokenizer(do_lower_case=UpperCAmelCase_)

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')							,							['HeLLo', '!', 'how', 'Are', 'yoU', '?'])





    def    SCREAMING_SNAKE_CASE_     (					self     :						Any):


         """simple docstring"""







         a					:							List[Any]							= BasicTokenizer(do_lower_case=UpperCAmelCase_							,							strip_accents=UpperCAmelCase_)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')							,							['HäLLo', '!', 'how', 'Are', 'yoU', '?'])





    def    SCREAMING_SNAKE_CASE_     (					self     :						Tuple):


         """simple docstring"""







         a					:							List[Any]							= BasicTokenizer(do_lower_case=UpperCAmelCase_							,							strip_accents=UpperCAmelCase_)

         self.assertListEqual(
             tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')							,							['HaLLo', '!', 'how', 'Are', 'yoU', '?'])





    def    SCREAMING_SNAKE_CASE_     (					self     :						str):


         """simple docstring"""







         a					:							List[Any]							= BasicTokenizer(do_lower_case=UpperCAmelCase_							,							never_split=['[UNK]'])

         self.assertListEqual(
             tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]')							,							['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'])





    def    SCREAMING_SNAKE_CASE_     (					self     :						Tuple):


         """simple docstring"""







         a					:							str							= ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

         a					:							Union[str, Any]							= {}
         for i, token in enumerate(UpperCAmelCase_):
              a					:							List[Any]							= i
         a					:							List[str]							= WordpieceTokenizer(vocab=UpperCAmelCase_							,							unk_token='[UNK]')

         self.assertListEqual(tokenizer.tokenize('')							,							[])

         self.assertListEqual(tokenizer.tokenize('unwanted running')							,							['un', '##want', '##ed', 'runn', '##ing'])

         self.assertListEqual(tokenizer.tokenize('unwantedX running')							,							['[UNK]', 'runn', '##ing'])





    def    SCREAMING_SNAKE_CASE_     (					self     :						List[Any]):


         """simple docstring"""







         self.assertTrue(_is_whitespace(' '))
         self.assertTrue(_is_whitespace('\t'))
         self.assertTrue(_is_whitespace('\r'))
         self.assertTrue(_is_whitespace('\n'))
         self.assertTrue(_is_whitespace('\u00A0'))

         self.assertFalse(_is_whitespace('A'))
         self.assertFalse(_is_whitespace('-'))





    def    SCREAMING_SNAKE_CASE_     (					self     :						Dict):


         """simple docstring"""







         self.assertTrue(_is_control('\u0005'))

         self.assertFalse(_is_control('A'))
         self.assertFalse(_is_control(' '))
         self.assertFalse(_is_control('\t'))
         self.assertFalse(_is_control('\r'))





    def    SCREAMING_SNAKE_CASE_     (					self     :						Optional[Any]):


         """simple docstring"""







         self.assertTrue(_is_punctuation('-'))
         self.assertTrue(_is_punctuation('$'))
         self.assertTrue(_is_punctuation('`'))
         self.assertTrue(_is_punctuation('.'))

         self.assertFalse(_is_punctuation('A'))
         self.assertFalse(_is_punctuation(' '))





    def    SCREAMING_SNAKE_CASE_     (					self     :						Dict):


         """simple docstring"""







         a					:							Any							= self.get_tokenizer()

         # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
         self.assertListEqual([tokenizer.tokenize(UpperCAmelCase_) for t in ['Test', '\xad', 'test']]							,							[['[UNK]'], [], ['[UNK]']])

         if self.test_rust_tokenizer:
              a					:							Tuple							= self.get_rust_tokenizer()
              self.assertListEqual(
                  [rust_tokenizer.tokenize(UpperCAmelCase_) for t in ['Test', '\xad', 'test']]							,							[['[UNK]'], [], ['[UNK]']])





    @slow
    def    SCREAMING_SNAKE_CASE_     (					self     :						List[str]):


         """simple docstring"""







         a					:							Dict							= self.tokenizer_class.from_pretrained('bert-base-uncased')

         a					:							str							= tokenizer.encode('sequence builders'							,							add_special_tokens=UpperCAmelCase_)
         a					:							Optional[Any]							= tokenizer.encode('multi-sequence build'							,							add_special_tokens=UpperCAmelCase_)

         a					:							str							= tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_)
         a					:							Optional[int]							= tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_							,							UpperCAmelCase_)

         assert encoded_sentence == [1_0_1] + text + [1_0_2]
         assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]





    def    SCREAMING_SNAKE_CASE_     (					self     :						List[str]):


         """simple docstring"""







         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                   a					:							Optional[Any]							= self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_							,							**UpperCAmelCase_)

                   a					:							str							= f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                   a					:							List[Any]							= tokenizer_r.encode_plus(
                       UpperCAmelCase_							,							return_attention_mask=UpperCAmelCase_							,							return_token_type_ids=UpperCAmelCase_							,							return_offsets_mapping=UpperCAmelCase_							,							add_special_tokens=UpperCAmelCase_							,							)

                   a					:							List[str]							= tokenizer_r.do_lower_case if hasattr(UpperCAmelCase_							,							'do_lower_case') else False
                   a					:							str							= (
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), 'A'),
                           ((1, 2), ','),
                           ((3, 5), 'na'),
                           ((5, 6), '##ï'),
                           ((6, 8), '##ve'),
                           ((9, 1_5), tokenizer_r.mask_token),
                           ((1_6, 2_1), 'Allen'),
                           ((2_1, 2_3), '##NL'),
                           ((2_3, 2_4), '##P'),
                           ((2_5, 3_3), 'sentence'),
                           ((3_3, 3_4), '.'),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), 'a'),
                           ((1, 2), ','),
                           ((3, 8), 'naive'),
                           ((9, 1_5), tokenizer_r.mask_token),
                           ((1_6, 2_1), 'allen'),
                           ((2_1, 2_3), '##nl'),
                           ((2_3, 2_4), '##p'),
                           ((2_5, 3_3), 'sentence'),
                           ((3_3, 3_4), '.'),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]							,							tokenizer_r.convert_ids_to_tokens(tokens['input_ids']))
                   self.assertEqual([e[0] for e in expected_results]							,							tokens['offset_mapping'])







    @slow
    def    SCREAMING_SNAKE_CASE_     (					self     :						Any):


         """simple docstring"""







         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                   a					:							Optional[Any]							= self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_							,							**UpperCAmelCase_)
                   a					:							List[Any]							= self.tokenizer_class.from_pretrained(UpperCAmelCase_							,							**UpperCAmelCase_)
                   a					:							Optional[Any]							= [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

                   a					:							int							= tokenizer_r.batch_encode_candidates(UpperCAmelCase_							,							max_length=1_0							,							return_tensors='np')
                   a					:							Union[str, Any]							= tokenizer_p.batch_encode_candidates(UpperCAmelCase_							,							max_length=1_0							,							return_tensors='np')

                   a					:							Optional[int]							= (2, 2, 1_0)

                   self.assertEqual(encoded_sentence_r['input_ids'].shape							,							UpperCAmelCase_)
                   self.assertEqual(encoded_sentence_r['attention_mask'].shape							,							UpperCAmelCase_)
                   self.assertEqual(encoded_sentence_r['token_type_ids'].shape							,							UpperCAmelCase_)

                   self.assertEqual(encoded_sentence_p['input_ids'].shape							,							UpperCAmelCase_)
                   self.assertEqual(encoded_sentence_p['attention_mask'].shape							,							UpperCAmelCase_)
                   self.assertEqual(encoded_sentence_p['token_type_ids'].shape							,							UpperCAmelCase_)


'''simple docstring'''





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCamelCase       :     Dict			=    logging.get_logger(__name__)

UpperCamelCase       :     Tuple			=    {
    """flaubert/flaubert_small_cased""": """https://huggingface.co/flaubert/flaubert_small_cased/resolve/main/config.json""",
    """flaubert/flaubert_base_uncased""": """https://huggingface.co/flaubert/flaubert_base_uncased/resolve/main/config.json""",
    """flaubert/flaubert_base_cased""": """https://huggingface.co/flaubert/flaubert_base_cased/resolve/main/config.json""",
    """flaubert/flaubert_large_cased""": """https://huggingface.co/flaubert/flaubert_large_cased/resolve/main/config.json""",
}



class   UpperCamelCase					(       a_       ):







    """simple docstring"""





    A     :    int				=			"flaubert"
    A     :    List[str]				=			{
        "hidden_size": "emb_dim",
        "num_attention_heads": "n_heads",
        "num_hidden_layers": "n_layers",
        "n_words": "vocab_size",  # For backward compatibility
    }





    def __init__(					self     :						str							,							UpperCAmelCase_     :						Optional[int]=False							,							UpperCAmelCase_     :						List[Any]=0.0							,							UpperCAmelCase_     :						List[Any]=3_0_1_4_5							,							UpperCAmelCase_     :						Union[str, Any]=2_0_4_8							,							UpperCAmelCase_     :						Tuple=1_2							,							UpperCAmelCase_     :						str=1_6							,							UpperCAmelCase_     :						Tuple=0.1							,							UpperCAmelCase_     :						str=0.1							,							UpperCAmelCase_     :						str=True							,							UpperCAmelCase_     :						Union[str, Any]=False							,							UpperCAmelCase_     :						Any=False							,							UpperCAmelCase_     :						str=False							,							UpperCAmelCase_     :						Tuple=1							,							UpperCAmelCase_     :						str=True							,							UpperCAmelCase_     :						Tuple=5_1_2							,							UpperCAmelCase_     :						List[str]=2_0_4_8**-0.5							,							UpperCAmelCase_     :						List[Any]=1e-12							,							UpperCAmelCase_     :						Tuple=0.02							,							UpperCAmelCase_     :						Any=0							,							UpperCAmelCase_     :						List[Any]=1							,							UpperCAmelCase_     :						str=2							,							UpperCAmelCase_     :						Optional[Any]=3							,							UpperCAmelCase_     :						List[str]=5							,							UpperCAmelCase_     :						str=True							,							UpperCAmelCase_     :						List[Any]="first"							,							UpperCAmelCase_     :						Any=True							,							UpperCAmelCase_     :						Union[str, Any]=None							,							UpperCAmelCase_     :						Union[str, Any]=True							,							UpperCAmelCase_     :						Any=0.1							,							UpperCAmelCase_     :						List[Any]=5							,							UpperCAmelCase_     :						List[Any]=5							,							UpperCAmelCase_     :						Union[str, Any]=0							,							UpperCAmelCase_     :						Optional[Any]=0							,							UpperCAmelCase_     :						List[Any]=2							,							UpperCAmelCase_     :						List[Any]=0							,							**UpperCAmelCase_     :						List[Any]							,							):


         """simple docstring"""







         a					:							Any							= pre_norm
         a					:							str							= layerdrop
         a					:							int							= vocab_size
         a					:							str							= emb_dim
         a					:							Any							= n_layers
         a					:							Dict							= n_heads
         a					:							Union[str, Any]							= dropout
         a					:							List[Any]							= attention_dropout
         a					:							List[str]							= gelu_activation
         a					:							int							= sinusoidal_embeddings
         a					:							Union[str, Any]							= causal
         a					:							str							= asm
         a					:							Tuple							= n_langs
         a					:							Optional[Any]							= use_lang_emb
         a					:							str							= layer_norm_eps
         a					:							Optional[Any]							= bos_index
         a					:							Union[str, Any]							= eos_index
         a					:							str							= pad_index
         a					:							List[str]							= unk_index
         a					:							Dict							= mask_index
         a					:							Optional[Any]							= is_encoder
         a					:							str							= max_position_embeddings
         a					:							Optional[Any]							= embed_init_std
         a					:							str							= init_std
         a					:							Union[str, Any]							= summary_type
         a					:							Tuple							= summary_use_proj
         a					:							Tuple							= summary_activation
         a					:							str							= summary_proj_to_labels
         a					:							List[str]							= summary_first_dropout
         a					:							Any							= start_n_top
         a					:							Optional[Any]							= end_n_top
         a					:							List[Any]							= mask_token_id
         a					:							Optional[Any]							= lang_id

         if "n_words" in kwargs:
              a					:							int							= kwargs['n_words']

         super().__init__(pad_token_id=UpperCAmelCase_							,							bos_token_id=UpperCAmelCase_							,							**UpperCAmelCase_)






class   UpperCamelCase					(       a_       ):







    """simple docstring"""





    @property
    def    SCREAMING_SNAKE_CASE_     (					self     :						str):


         """simple docstring"""







         if self.task == "multiple-choice":
              a					:							int							= {0: 'batch', 1: 'choice', 2: 'sequence'}
         else:
              a					:							Tuple							= {0: 'batch', 1: 'sequence'}
         return OrderedDict(
             [
                 ('input_ids', dynamic_axis),
                 ('attention_mask', dynamic_axis),
             ])

