




"""simple docstring"""

import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 							lowercase   (     a_  ,						unittest.TestCase):
				__lowerCAmelCase     :     Optional[int]								= RealmTokenizer
				__lowerCAmelCase     :     Optional[Any]								= RealmTokenizerFast
				__lowerCAmelCase     :     Optional[int]								= True
				__lowerCAmelCase     :     Dict								= True
				__lowerCAmelCase     :     Union[str, Any]								= filter_non_english







				def        a_	(				self :       Optional[int]							):



					"""simple docstring"""

					super().setUp()

					A_	:					Optional[Any]						 =		[
					    '[UNK]',
					    '[CLS]',
					    '[SEP]',
					    '[PAD]',
					    '[MASK]',
					    'want',
					    '##want',
					    '##ed',
					    'wa',
					    'un',
					    'runn',
					    '##ing',
					    ',',
					    'low',
					    'lowest',
					]
					A_	:					Tuple						 =		os.path.join(self.tmpdirname   ,  VOCAB_FILES_NAMES['''vocab_file''']							)
					with open(self.vocab_file   ,  '''w'''   ,  encoding='''utf-8'''							) as vocab_writer:
						vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)







				def        a_	(				self :       Dict   ,  _lowerCamelCase :       str							):



					"""simple docstring"""

					A_	:					int						 =		'UNwant\u00E9d,running'
					A_	:					List[str]						 =		'unwanted, running'
					return input_text, output_text







				def        a_	(				self :       Any							):



					"""simple docstring"""

					A_	:					Optional[int]						 =		self.tokenizer_class(self.vocab_file							)

					A_	:					Tuple						 =		tokenizer.tokenize('''UNwant\u00E9d,running'''							)
					self.assertListEqual(UpperCAmelCase_   ,  ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']							)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_							)   ,  [9, 6, 7, 12, 10, 11]							)







				def        a_	(				self :       str							):



					"""simple docstring"""

					if not self.test_rust_tokenizer:
						return

					A_	:					Optional[int]						 =		self.get_tokenizer()
					A_	:					List[str]						 =		self.get_rust_tokenizer()

					A_	:					int						 =		'UNwant\u00E9d,running'

					A_	:					str						 =		tokenizer.tokenize(UpperCAmelCase_							)
					A_	:					List[str]						 =		rust_tokenizer.tokenize(UpperCAmelCase_							)
					self.assertListEqual(UpperCAmelCase_   ,  UpperCAmelCase_							)

					A_	:					Tuple						 =		tokenizer.encode(UpperCAmelCase_   ,  add_special_tokens=UpperCAmelCase_							)
					A_	:					List[Any]						 =		rust_tokenizer.encode(UpperCAmelCase_   ,  add_special_tokens=UpperCAmelCase_							)
					self.assertListEqual(UpperCAmelCase_   ,  UpperCAmelCase_							)

					A_	:					int						 =		self.get_rust_tokenizer()
					A_	:					List[str]						 =		tokenizer.encode(UpperCAmelCase_							)
					A_	:					int						 =		rust_tokenizer.encode(UpperCAmelCase_							)
					self.assertListEqual(UpperCAmelCase_   ,  UpperCAmelCase_							)

					# With lower casing
					A_	:					List[str]						 =		self.get_tokenizer(do_lower_case=UpperCAmelCase_							)
					A_	:					Optional[int]						 =		self.get_rust_tokenizer(do_lower_case=UpperCAmelCase_							)

					A_	:					Optional[Any]						 =		'UNwant\u00E9d,running'

					A_	:					List[Any]						 =		tokenizer.tokenize(UpperCAmelCase_							)
					A_	:					List[str]						 =		rust_tokenizer.tokenize(UpperCAmelCase_							)
					self.assertListEqual(UpperCAmelCase_   ,  UpperCAmelCase_							)

					A_	:					List[Any]						 =		tokenizer.encode(UpperCAmelCase_   ,  add_special_tokens=UpperCAmelCase_							)
					A_	:					Any						 =		rust_tokenizer.encode(UpperCAmelCase_   ,  add_special_tokens=UpperCAmelCase_							)
					self.assertListEqual(UpperCAmelCase_   ,  UpperCAmelCase_							)

					A_	:					Dict						 =		self.get_rust_tokenizer()
					A_	:					Optional[Any]						 =		tokenizer.encode(UpperCAmelCase_							)
					A_	:					Optional[int]						 =		rust_tokenizer.encode(UpperCAmelCase_							)
					self.assertListEqual(UpperCAmelCase_   ,  UpperCAmelCase_							)







				def        a_	(				self :       Tuple							):



					"""simple docstring"""

					A_	:					Any						 =		BasicTokenizer()

					self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''							)   ,  ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']							)







				def        a_	(				self :       Union[str, Any]							):



					"""simple docstring"""

					A_	:					int						 =		BasicTokenizer(do_lower_case=UpperCAmelCase_							)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)   ,  ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)   ,  ['''hello''']							)







				def        a_	(				self :       Tuple							):



					"""simple docstring"""

					A_	:					Dict						 =		BasicTokenizer(do_lower_case=UpperCAmelCase_   ,  strip_accents=UpperCAmelCase_							)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)   ,  ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)   ,  ['''h\u00E9llo''']							)







				def        a_	(				self :       List[Any]							):



					"""simple docstring"""

					A_	:					str						 =		BasicTokenizer(do_lower_case=UpperCAmelCase_   ,  strip_accents=UpperCAmelCase_							)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)   ,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)   ,  ['''hello''']							)







				def        a_	(				self :       Any							):



					"""simple docstring"""

					A_	:					Optional[Any]						 =		BasicTokenizer(do_lower_case=UpperCAmelCase_							)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)   ,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
					self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)   ,  ['''hello''']							)







				def        a_	(				self :       List[Any]							):



					"""simple docstring"""

					A_	:					Optional[Any]						 =		BasicTokenizer(do_lower_case=UpperCAmelCase_							)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)   ,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)







				def        a_	(				self :       Any							):



					"""simple docstring"""

					A_	:					List[Any]						 =		BasicTokenizer(do_lower_case=UpperCAmelCase_   ,  strip_accents=UpperCAmelCase_							)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)   ,  ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)







				def        a_	(				self :       Tuple							):



					"""simple docstring"""

					A_	:					List[Any]						 =		BasicTokenizer(do_lower_case=UpperCAmelCase_   ,  strip_accents=UpperCAmelCase_							)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)   ,  ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)







				def        a_	(				self :       str							):



					"""simple docstring"""

					A_	:					List[Any]						 =		BasicTokenizer(do_lower_case=UpperCAmelCase_   ,  never_split=['''[UNK]''']							)

					self.assertListEqual(
					    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''							)   ,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']							)







				def        a_	(				self :       Tuple							):



					"""simple docstring"""

					A_	:					str						 =		['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

					A_	:					Union[str, Any]						 =		{}
					for i, token in enumerate(UpperCAmelCase_							):
						A_	:					List[Any]						 =		i
					A_	:					List[str]						 =		WordpieceTokenizer(vocab=UpperCAmelCase_   ,  unk_token='''[UNK]'''							)

					self.assertListEqual(tokenizer.tokenize(''''''							)   ,  []							)

					self.assertListEqual(tokenizer.tokenize('''unwanted running'''							)   ,  ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']							)

					self.assertListEqual(tokenizer.tokenize('''unwantedX running'''							)   ,  ['''[UNK]''', '''runn''', '''##ing''']							)







				def        a_	(				self :       List[Any]							):



					"""simple docstring"""

					self.assertTrue(_is_whitespace(''' '''							)							)
					self.assertTrue(_is_whitespace('''\t'''							)							)
					self.assertTrue(_is_whitespace('''\r'''							)							)
					self.assertTrue(_is_whitespace('''\n'''							)							)
					self.assertTrue(_is_whitespace('''\u00A0'''							)							)

					self.assertFalse(_is_whitespace('''A'''							)							)
					self.assertFalse(_is_whitespace('''-'''							)							)







				def        a_	(				self :       Dict							):



					"""simple docstring"""

					self.assertTrue(_is_control('''\u0005'''							)							)

					self.assertFalse(_is_control('''A'''							)							)
					self.assertFalse(_is_control(''' '''							)							)
					self.assertFalse(_is_control('''\t'''							)							)
					self.assertFalse(_is_control('''\r'''							)							)







				def        a_	(				self :       Optional[Any]							):



					"""simple docstring"""

					self.assertTrue(_is_punctuation('''-'''							)							)
					self.assertTrue(_is_punctuation('''$'''							)							)
					self.assertTrue(_is_punctuation('''`'''							)							)
					self.assertTrue(_is_punctuation('''.'''							)							)

					self.assertFalse(_is_punctuation('''A'''							)							)
					self.assertFalse(_is_punctuation(''' '''							)							)







				def        a_	(				self :       Dict							):



					"""simple docstring"""

					A_	:					Any						 =		self.get_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(UpperCAmelCase_							) for t in ['''Test''', '''\xad''', '''test''']]   ,  [['''[UNK]'''], [], ['''[UNK]''']]							)

					if self.test_rust_tokenizer:
						A_	:					Tuple						 =		self.get_rust_tokenizer()
						self.assertListEqual(
						    [rust_tokenizer.tokenize(UpperCAmelCase_							) for t in ['''Test''', '''\xad''', '''test''']]   ,  [['''[UNK]'''], [], ['''[UNK]''']]							)







				@slow
				def        a_	(				self :       List[str]							):



					"""simple docstring"""

					A_	:					Dict						 =		self.tokenizer_class.from_pretrained('''bert-base-uncased'''							)

					A_	:					str						 =		tokenizer.encode('''sequence builders'''   ,  add_special_tokens=UpperCAmelCase_							)
					A_	:					Optional[Any]						 =		tokenizer.encode('''multi-sequence build'''   ,  add_special_tokens=UpperCAmelCase_							)

					A_	:					str						 =		tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_							)
					A_	:					Optional[int]						 =		tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_   ,  UpperCAmelCase_							)

					assert encoded_sentence == [1_01] + text + [1_02]
					assert encoded_pair == [1_01] + text + [1_02] + text_a + [1_02]







				def        a_	(				self :       List[str]							):



					"""simple docstring"""

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
							A_	:					Optional[Any]						 =		self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_   ,  **UpperCAmelCase_							)

							A_	:					str						 =		F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
							A_	:					List[Any]						 =		tokenizer_r.encode_plus(
							    UpperCAmelCase_   ,  return_attention_mask=UpperCAmelCase_   ,  return_token_type_ids=UpperCAmelCase_   ,  return_offsets_mapping=UpperCAmelCase_   ,  add_special_tokens=UpperCAmelCase_   ,  )

							A_	:					List[str]						 =		tokenizer_r.do_lower_case if hasattr(UpperCAmelCase_   ,  '''do_lower_case'''							) else False
							A_	:					str						 =		(
							    [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), 'A'),
							        ((1, 2), ','),
							        ((3, 5), 'na'),
							        ((5, 6), '##ï'),
							        ((6, 8), '##ve'),
							        ((9, 15), tokenizer_r.mask_token),
							        ((16, 21), 'Allen'),
							        ((21, 23), '##NL'),
							        ((23, 24), '##P'),
							        ((25, 33), 'sentence'),
							        ((33, 34), '.'),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							    if not do_lower_case
							    else [
							        ((0, 0), tokenizer_r.cls_token),
							        ((0, 1), 'a'),
							        ((1, 2), ','),
							        ((3, 8), 'naive'),
							        ((9, 15), tokenizer_r.mask_token),
							        ((16, 21), 'allen'),
							        ((21, 23), '##nl'),
							        ((23, 24), '##p'),
							        ((25, 33), 'sentence'),
							        ((33, 34), '.'),
							        ((0, 0), tokenizer_r.sep_token),
							    ]
							)

							self.assertEqual(
							    [e[1] for e in expected_results]   ,  tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']							)							)
							self.assertEqual([e[0] for e in expected_results]   ,  tokens['''offset_mapping''']							)







				@slow
				def        a_	(				self :       Any							):



					"""simple docstring"""

					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
						with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
							A_	:					Optional[Any]						 =		self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_   ,  **UpperCAmelCase_							)
							A_	:					List[Any]						 =		self.tokenizer_class.from_pretrained(UpperCAmelCase_   ,  **UpperCAmelCase_							)
							A_	:					Optional[Any]						 =		[['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

							A_	:					int						 =		tokenizer_r.batch_encode_candidates(UpperCAmelCase_   ,  max_length=10   ,  return_tensors='''np'''							)
							A_	:					Union[str, Any]						 =		tokenizer_p.batch_encode_candidates(UpperCAmelCase_   ,  max_length=10   ,  return_tensors='''np'''							)

							A_	:					Optional[int]						 =		(2, 2, 10)

							self.assertEqual(encoded_sentence_r['''input_ids'''].shape   ,  UpperCAmelCase_							)
							self.assertEqual(encoded_sentence_r['''attention_mask'''].shape   ,  UpperCAmelCase_							)
							self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape   ,  UpperCAmelCase_							)

							self.assertEqual(encoded_sentence_p['''input_ids'''].shape   ,  UpperCAmelCase_							)
							self.assertEqual(encoded_sentence_p['''attention_mask'''].shape   ,  UpperCAmelCase_							)
							self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape   ,  UpperCAmelCase_							)






'''simple docstring'''





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCamelCase       :     Dict			=    logging.get_logger(__name__)

UpperCamelCase       :     Tuple			=    {
    """flaubert/flaubert_small_cased""": """https://huggingface.co/flaubert/flaubert_small_cased/resolve/main/config.json""",
    """flaubert/flaubert_base_uncased""": """https://huggingface.co/flaubert/flaubert_base_uncased/resolve/main/config.json""",
    """flaubert/flaubert_base_cased""": """https://huggingface.co/flaubert/flaubert_base_cased/resolve/main/config.json""",
    """flaubert/flaubert_large_cased""": """https://huggingface.co/flaubert/flaubert_large_cased/resolve/main/config.json""",
}



class   UpperCamelCase					(       a_       ):







    """simple docstring"""





    A     :    int				=			"flaubert"
    A     :    List[str]				=			{
        "hidden_size": "emb_dim",
        "num_attention_heads": "n_heads",
        "num_hidden_layers": "n_layers",
        "n_words": "vocab_size",  # For backward compatibility
    }





    def __init__(					self     :						str							,							UpperCAmelCase_     :						Optional[int]=False							,							UpperCAmelCase_     :						List[Any]=0.0							,							UpperCAmelCase_     :						List[Any]=3_0_1_4_5							,							UpperCAmelCase_     :						Union[str, Any]=2_0_4_8							,							UpperCAmelCase_     :						Tuple=1_2							,							UpperCAmelCase_     :						str=1_6							,							UpperCAmelCase_     :						Tuple=0.1							,							UpperCAmelCase_     :						str=0.1							,							UpperCAmelCase_     :						str=True							,							UpperCAmelCase_     :						Union[str, Any]=False							,							UpperCAmelCase_     :						Any=False							,							UpperCAmelCase_     :						str=False							,							UpperCAmelCase_     :						Tuple=1							,							UpperCAmelCase_     :						str=True							,							UpperCAmelCase_     :						Tuple=5_1_2							,							UpperCAmelCase_     :						List[str]=2_0_4_8**-0.5							,							UpperCAmelCase_     :						List[Any]=1e-12							,							UpperCAmelCase_     :						Tuple=0.02							,							UpperCAmelCase_     :						Any=0							,							UpperCAmelCase_     :						List[Any]=1							,							UpperCAmelCase_     :						str=2							,							UpperCAmelCase_     :						Optional[Any]=3							,							UpperCAmelCase_     :						List[str]=5							,							UpperCAmelCase_     :						str=True							,							UpperCAmelCase_     :						List[Any]="first"							,							UpperCAmelCase_     :						Any=True							,							UpperCAmelCase_     :						Union[str, Any]=None							,							UpperCAmelCase_     :						Union[str, Any]=True							,							UpperCAmelCase_     :						Any=0.1							,							UpperCAmelCase_     :						List[Any]=5							,							UpperCAmelCase_     :						List[Any]=5							,							UpperCAmelCase_     :						Union[str, Any]=0							,							UpperCAmelCase_     :						Optional[Any]=0							,							UpperCAmelCase_     :						List[Any]=2							,							UpperCAmelCase_     :						List[Any]=0							,							**UpperCAmelCase_     :						List[Any]							,							):


         """simple docstring"""







         a					:							Any							= pre_norm
         a					:							str							= layerdrop
         a					:							int							= vocab_size
         a					:							str							= emb_dim
         a					:							Any							= n_layers
         a					:							Dict							= n_heads
         a					:							Union[str, Any]							= dropout
         a					:							List[Any]							= attention_dropout
         a					:							List[str]							= gelu_activation
         a					:							int							= sinusoidal_embeddings
         a					:							Union[str, Any]							= causal
         a					:							str							= asm
         a					:							Tuple							= n_langs
         a					:							Optional[Any]							= use_lang_emb
         a					:							str							= layer_norm_eps
         a					:							Optional[Any]							= bos_index
         a					:							Union[str, Any]							= eos_index
         a					:							str							= pad_index
         a					:							List[str]							= unk_index
         a					:							Dict							= mask_index
         a					:							Optional[Any]							= is_encoder
         a					:							str							= max_position_embeddings
         a					:							Optional[Any]							= embed_init_std
         a					:							str							= init_std
         a					:							Union[str, Any]							= summary_type
         a					:							Tuple							= summary_use_proj
         a					:							Tuple							= summary_activation
         a					:							str							= summary_proj_to_labels
         a					:							List[str]							= summary_first_dropout
         a					:							Any							= start_n_top
         a					:							Optional[Any]							= end_n_top
         a					:							List[Any]							= mask_token_id
         a					:							Optional[Any]							= lang_id

         if "n_words" in kwargs:
              a					:							int							= kwargs['n_words']

         super().__init__(pad_token_id=UpperCAmelCase_							,							bos_token_id=UpperCAmelCase_							,							**UpperCAmelCase_)






class   UpperCamelCase					(       a_       ):







    """simple docstring"""





    @property
    def    SCREAMING_SNAKE_CASE_     (					self     :						str):


         """simple docstring"""







         if self.task == "multiple-choice":
              a					:							int							= {0: 'batch', 1: 'choice', 2: 'sequence'}
         else:
              a					:							Tuple							= {0: 'batch', 1: 'sequence'}
         return OrderedDict(
             [
                 ('input_ids', dynamic_axis),
                 ('attention_mask', dynamic_axis),
             ])

