import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 					lowercase__						(   UpperCamelCase_	,							unittest.TestCase):
						UpperCamelCase_     =		None
						UpperCamelCase_     =		BloomTokenizerFast
						UpperCamelCase_     =		BloomTokenizerFast
						UpperCamelCase_     =		True
						UpperCamelCase_     =		False
						UpperCamelCase_     =		"""tokenizer_file"""
						UpperCamelCase_     =		{"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}


						def        __A       (       self	:				Optional[int]						):





												'''simple docstring'''





												super().setUp()
												SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							BloomTokenizerFast.from_pretrained('''bigscience/tokenizer'''						)
												tokenizer.save_pretrained(self.tmpdirname						)


						def        __A       (       self	:				Optional[Any]    ,		**UpperCamelCase__	:				Union[str, Any]						):





												'''simple docstring'''





												kwargs.update(self.special_tokens_map						)
												return BloomTokenizerFast.from_pretrained(self.tmpdirname    ,		**UpperCamelCase__						)


						def        __A       (       self	:				Any						):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:    Optional[Any]							=							self.get_rust_tokenizer()

												SCREAMING_SNAKE_CASE							:    str							=							['''The quick brown fox</s>''', '''jumps over the lazy dog</s>''']
												SCREAMING_SNAKE_CASE							:    List[Any]							=							[[2175, 2_3714, 7_3173, 14_4252, 2], [77, 13_2619, 3478, 368, 10_9586, 3_5433, 2]]

												SCREAMING_SNAKE_CASE							:    Tuple							=							tokenizer.batch_encode_plus(UpperCamelCase__						)['''input_ids''']
												self.assertListEqual(UpperCamelCase__    ,		UpperCamelCase__						)

												SCREAMING_SNAKE_CASE							:    Optional[int]							=							tokenizer.batch_decode(UpperCamelCase__						)
												self.assertListEqual(UpperCamelCase__    ,		UpperCamelCase__						)


						def        __A       (       self	:				List[str]    ,		UpperCamelCase__	:				Optional[Any]=6						):





												'''simple docstring'''





												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""						):
																								SCREAMING_SNAKE_CASE							:    Any							=							self.rust_tokenizer_class.from_pretrained(UpperCamelCase__    ,		**UpperCamelCase__						)
																								# tokenizer_r.pad_token = None # Hotfixing padding = None
																								# Simple input
																								SCREAMING_SNAKE_CASE							:    str							=							'''This is a simple input'''
																								SCREAMING_SNAKE_CASE							:    str							=							['''This is a simple input 1''', '''This is a simple input 2''']
																								SCREAMING_SNAKE_CASE							:    List[Any]							=							('''This is a simple input''', '''This is a pair''')
																								SCREAMING_SNAKE_CASE							:    Optional[Any]							=							[
																								    ('''This is a simple input 1''', '''This is a simple input 2'''),
																								    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																								]

																								# Simple input tests
																								try:
																														tokenizer_r.encode(UpperCamelCase__    ,		max_length=UpperCamelCase__						)
																														tokenizer_r.encode_plus(UpperCamelCase__    ,		max_length=UpperCamelCase__						)

																														tokenizer_r.batch_encode_plus(UpperCamelCase__    ,		max_length=UpperCamelCase__						)
																														tokenizer_r.encode(UpperCamelCase__    ,		max_length=UpperCamelCase__						)
																														tokenizer_r.batch_encode_plus(UpperCamelCase__    ,		max_length=UpperCamelCase__						)
																								except ValueError:
																														self.fail('''Bloom Tokenizer should be able to deal with padding'''						)

																								SCREAMING_SNAKE_CASE							:    Tuple							=							None  # Hotfixing padding = None
																								self.assertRaises(UpperCamelCase__    ,		tokenizer_r.encode    ,		UpperCamelCase__    ,		max_length=UpperCamelCase__    ,		padding='''max_length'''						)

																								# Simple input
																								self.assertRaises(UpperCamelCase__    ,		tokenizer_r.encode_plus    ,		UpperCamelCase__    ,		max_length=UpperCamelCase__    ,		padding='''max_length'''						)

																								# Simple input
																								self.assertRaises(
																								    UpperCamelCase__    ,		tokenizer_r.batch_encode_plus    ,		UpperCamelCase__    ,		max_length=UpperCamelCase__    ,		padding='''max_length'''    ,		)

																								# Pair input
																								self.assertRaises(UpperCamelCase__    ,		tokenizer_r.encode    ,		UpperCamelCase__    ,		max_length=UpperCamelCase__    ,		padding='''max_length'''						)

																								# Pair input
																								self.assertRaises(UpperCamelCase__    ,		tokenizer_r.encode_plus    ,		UpperCamelCase__    ,		max_length=UpperCamelCase__    ,		padding='''max_length'''						)

																								# Pair input
																								self.assertRaises(
																								    UpperCamelCase__    ,		tokenizer_r.batch_encode_plus    ,		UpperCamelCase__    ,		max_length=UpperCamelCase__    ,		padding='''max_length'''    ,		)


						def        __A       (       self	:				List[Any]						):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:    Dict							=							self.get_rust_tokenizer()
												SCREAMING_SNAKE_CASE							:    List[Any]							=							load_dataset('''xnli'''    ,		'''all_languages'''    ,		split='''test'''    ,		streaming=UpperCamelCase__						)

												SCREAMING_SNAKE_CASE							:    int							=							next(iter(UpperCamelCase__						)						)['''premise''']  # pick up one data
												SCREAMING_SNAKE_CASE							:    int							=							list(sample_data.values()						)

												SCREAMING_SNAKE_CASE							:    int							=							list(map(tokenizer.encode    ,		UpperCamelCase__						)						)
												SCREAMING_SNAKE_CASE							:    Optional[Any]							=							[tokenizer.decode(UpperCamelCase__    ,		clean_up_tokenization_spaces=UpperCamelCase__						) for x in output_tokens]
												self.assertListEqual(UpperCamelCase__    ,		UpperCamelCase__						)



						def        __A       (       self	:				List[str]						):





												'''simple docstring'''





												self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map						)    ,		1						)
												self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()						)[0]						)    ,		1						)






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCamelCase		:       Any									=     logging.get_logger(__name__)

__UpperCamelCase		:       Optional[int]									=     {
    'facebook/nllb-moe-54B': 'https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json',
}





class 					lowercase__						(   UpperCamelCase_):
						UpperCamelCase_     =		"""nllb-moe"""
						UpperCamelCase_     =		["""past_key_values"""]
						UpperCamelCase_     =		{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}


						def __init__(       self	:				List[str]    ,		UpperCamelCase__	:				List[str]=12_8112    ,		UpperCamelCase__	:				str=1024    ,		UpperCamelCase__	:				Optional[int]=12    ,		UpperCamelCase__	:				Union[str, Any]=4096    ,		UpperCamelCase__	:				Optional[Any]=16    ,		UpperCamelCase__	:				Optional[Any]=12    ,		UpperCamelCase__	:				str=4096    ,		UpperCamelCase__	:				Dict=16    ,		UpperCamelCase__	:				Any=0.05    ,		UpperCamelCase__	:				Any=0.05    ,		UpperCamelCase__	:				Optional[Any]=True    ,		UpperCamelCase__	:				Optional[Any]=True    ,		UpperCamelCase__	:				List[Any]="relu"    ,		UpperCamelCase__	:				Union[str, Any]=1024    ,		UpperCamelCase__	:				Optional[int]=0.1    ,		UpperCamelCase__	:				Any=0.1    ,		UpperCamelCase__	:				Optional[Any]=0.0    ,		UpperCamelCase__	:				str=0.02    ,		UpperCamelCase__	:				Tuple=2    ,		UpperCamelCase__	:				List[str]=True    ,		UpperCamelCase__	:				Dict=False    ,		UpperCamelCase__	:				Any="float32"    ,		UpperCamelCase__	:				List[Any]=False    ,		UpperCamelCase__	:				Tuple=128    ,		UpperCamelCase__	:				Tuple=64    ,		UpperCamelCase__	:				List[str]=4    ,		UpperCamelCase__	:				Union[str, Any]=4    ,		UpperCamelCase__	:				Dict=0.001    ,		UpperCamelCase__	:				Optional[Any]=0.001    ,		UpperCamelCase__	:				Optional[Any]="all"    ,		UpperCamelCase__	:				Tuple=False    ,		UpperCamelCase__	:				List[Any]=False    ,		UpperCamelCase__	:				List[Any]=1.0    ,		UpperCamelCase__	:				str=0.2    ,		UpperCamelCase__	:				List[Any]=1    ,		UpperCamelCase__	:				List[Any]=0    ,		UpperCamelCase__	:				Optional[int]=2    ,		UpperCamelCase__	:				Tuple=False    ,		**UpperCamelCase__	:				Union[str, Any]    ,		):





												'''simple docstring'''





												SCREAMING_SNAKE_CASE							:    List[str]							=							vocab_size
												SCREAMING_SNAKE_CASE							:    Dict							=							max_position_embeddings
												SCREAMING_SNAKE_CASE							:    int							=							d_model
												SCREAMING_SNAKE_CASE							:    Any							=							encoder_ffn_dim
												SCREAMING_SNAKE_CASE							:    List[Any]							=							encoder_layers
												SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							encoder_attention_heads
												SCREAMING_SNAKE_CASE							:    List[str]							=							decoder_ffn_dim
												SCREAMING_SNAKE_CASE							:    Dict							=							decoder_layers
												SCREAMING_SNAKE_CASE							:    List[Any]							=							decoder_attention_heads
												SCREAMING_SNAKE_CASE							:    List[str]							=							dropout
												SCREAMING_SNAKE_CASE							:    Any							=							attention_dropout
												SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							activation_dropout
												SCREAMING_SNAKE_CASE							:    List[Any]							=							activation_function
												SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							init_std
												SCREAMING_SNAKE_CASE							:    int							=							encoder_layerdrop
												SCREAMING_SNAKE_CASE							:    List[Any]							=							decoder_layerdrop
												SCREAMING_SNAKE_CASE							:    Any							=							use_cache
												SCREAMING_SNAKE_CASE							:    str							=							encoder_layers
												SCREAMING_SNAKE_CASE							:    List[Any]							=							scale_embedding  # scale factor will be sqrt(d_model) if True
												SCREAMING_SNAKE_CASE							:    List[str]							=							router_z_loss_coef
												SCREAMING_SNAKE_CASE							:    List[str]							=							router_aux_loss_coef
												SCREAMING_SNAKE_CASE							:    int							=							decoder_sparse_step
												SCREAMING_SNAKE_CASE							:    Optional[int]							=							encoder_sparse_step
												SCREAMING_SNAKE_CASE							:    List[str]							=							num_experts
												SCREAMING_SNAKE_CASE							:    int							=							expert_capacity
												SCREAMING_SNAKE_CASE							:    Any							=							router_bias
												if router_dtype not in ["float32", "float16", "bfloat16"]:
																		raise ValueError(f"""`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"""						)
												SCREAMING_SNAKE_CASE							:    str							=							router_dtype

												SCREAMING_SNAKE_CASE							:    List[Any]							=							router_ignore_padding_tokens
												SCREAMING_SNAKE_CASE							:    int							=							batch_prioritized_routing
												SCREAMING_SNAKE_CASE							:    str							=							second_expert_policy
												SCREAMING_SNAKE_CASE							:    Optional[Any]							=							normalize_router_prob_before_dropping
												SCREAMING_SNAKE_CASE							:    Optional[Any]							=							moe_eval_capacity_token_fraction
												SCREAMING_SNAKE_CASE							:    Optional[int]							=							moe_token_dropout
												SCREAMING_SNAKE_CASE							:    Optional[int]							=							output_router_logits
												super().__init__(
												    pad_token_id=UpperCamelCase__    ,		bos_token_id=UpperCamelCase__    ,		eos_token_id=UpperCamelCase__    ,		is_encoder_decoder=UpperCamelCase__    ,		decoder_start_token_id=UpperCamelCase__    ,		**UpperCamelCase__    ,		)





