





"""simple docstring"""
import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 					_SCREAMING_SNAKE_CASE(   A    ,	unittest.TestCase  ):
       SCREAMING_SNAKE_CASE_   :      List[Any]							=  None
       SCREAMING_SNAKE_CASE_   :      str							=  BloomTokenizerFast
       SCREAMING_SNAKE_CASE_   :      List[str]							=  BloomTokenizerFast
       SCREAMING_SNAKE_CASE_   :      Optional[int]							=  True
       SCREAMING_SNAKE_CASE_   :      Dict							=  False
       SCREAMING_SNAKE_CASE_   :      int							=  '''tokenizer_file'''
       SCREAMING_SNAKE_CASE_   :      Union[str, Any]							=  {'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}




       def     _UpperCamelCase     (						self							)			->      Tuple:





             """simple docstring"""






             super().setUp()
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  BloomTokenizerFast.from_pretrained('''bigscience/tokenizer'''							)
             tokenizer.save_pretrained(self.tmpdirname							)




       def     _UpperCamelCase     (						self		,**SCREAMING_SNAKE_CASE__							)			->      Any:





             """simple docstring"""






             kwargs.update(self.special_tokens_map							)
             return BloomTokenizerFast.from_pretrained(self.tmpdirname		,**SCREAMING_SNAKE_CASE__							)




       def     _UpperCamelCase     (						self							)			->      Union[str, Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Dict						=  self.get_rust_tokenizer()

             __SCREAMING_SNAKE_CASE   :Dict						=  ['''The quick brown fox</s>''', '''jumps over the lazy dog</s>''']
             __SCREAMING_SNAKE_CASE   :List[Any]						=  [[21_75, 2_37_14, 7_31_73, 14_42_52, 2], [77, 13_26_19, 34_78, 3_68, 10_95_86, 3_54_33, 2]]

             __SCREAMING_SNAKE_CASE   :List[str]						=  tokenizer.batch_encode_plus(SCREAMING_SNAKE_CASE__							)['''input_ids''']
             self.assertListEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)

             __SCREAMING_SNAKE_CASE   :Dict						=  tokenizer.batch_decode(SCREAMING_SNAKE_CASE__							)
             self.assertListEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__=6							)			->      Union[str, Any]:





             """simple docstring"""






             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
                         __SCREAMING_SNAKE_CASE   :Optional[int]						=  self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__		,**SCREAMING_SNAKE_CASE__							)
                         # tokenizer_r.pad_token = None # Hotfixing padding = None
                         # Simple input
                         __SCREAMING_SNAKE_CASE   :Optional[Any]						=  '''This is a simple input'''
                         __SCREAMING_SNAKE_CASE   :Optional[int]						=  ['''This is a simple input 1''', '''This is a simple input 2''']
                         __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  ('''This is a simple input''', '''This is a pair''')
                         __SCREAMING_SNAKE_CASE   :List[str]						=  [
                             ('''This is a simple input 1''', '''This is a simple input 2'''),
                             ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                         ]

                         # Simple input tests
                         try:
                               tokenizer_r.encode(SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__							)
                               tokenizer_r.encode_plus(SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__							)

                               tokenizer_r.batch_encode_plus(SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__							)
                               tokenizer_r.encode(SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__							)
                               tokenizer_r.batch_encode_plus(SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__							)
                         except ValueError:
                               self.fail('''Bloom Tokenizer should be able to deal with padding'''							)

                         __SCREAMING_SNAKE_CASE   :Dict						=  None  # Hotfixing padding = None
                         self.assertRaises(SCREAMING_SNAKE_CASE__		,tokenizer_r.encode		,SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__		,padding='''max_length'''							)

                         # Simple input
                         self.assertRaises(SCREAMING_SNAKE_CASE__		,tokenizer_r.encode_plus		,SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__		,padding='''max_length'''							)

                         # Simple input
                         self.assertRaises(
                             SCREAMING_SNAKE_CASE__		,tokenizer_r.batch_encode_plus		,SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__		,padding='''max_length'''		,)

                         # Pair input
                         self.assertRaises(SCREAMING_SNAKE_CASE__		,tokenizer_r.encode		,SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__		,padding='''max_length'''							)

                         # Pair input
                         self.assertRaises(SCREAMING_SNAKE_CASE__		,tokenizer_r.encode_plus		,SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__		,padding='''max_length'''							)

                         # Pair input
                         self.assertRaises(
                             SCREAMING_SNAKE_CASE__		,tokenizer_r.batch_encode_plus		,SCREAMING_SNAKE_CASE__		,max_length=SCREAMING_SNAKE_CASE__		,padding='''max_length'''		,)




       def     _UpperCamelCase     (						self							)			->      List[str]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :str						=  self.get_rust_tokenizer()
             __SCREAMING_SNAKE_CASE   :int						=  load_dataset('''xnli'''		,'''all_languages'''		,split='''test'''		,streaming=SCREAMING_SNAKE_CASE__							)

             __SCREAMING_SNAKE_CASE   :Tuple						=  next(iter(SCREAMING_SNAKE_CASE__							)							)['''premise''']  # pick up one data
             __SCREAMING_SNAKE_CASE   :List[str]						=  list(sample_data.values()							)

             __SCREAMING_SNAKE_CASE   :int						=  list(map(tokenizer.encode		,SCREAMING_SNAKE_CASE__							)							)
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  [tokenizer.decode(SCREAMING_SNAKE_CASE__		,clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE__							) for x in output_tokens]
             self.assertListEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)




       def     _UpperCamelCase     (						self							)			->      int:





             """simple docstring"""






             self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map							)		,1							)
             self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()							)[0]							)		,1							)






"""simple docstring"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_								=		logging.get_logger(__name__)

lowerCamelCase_								=		{}






class 					_SCREAMING_SNAKE_CASE(   A  ):
       SCREAMING_SNAKE_CASE_   :      List[Any]							=  '''llama'''
       SCREAMING_SNAKE_CASE_   :      Optional[int]							=  ['''past_key_values''']




       def __init__(						self		,SCREAMING_SNAKE_CASE__=3_20_00		,SCREAMING_SNAKE_CASE__=40_96		,SCREAMING_SNAKE_CASE__=1_10_08		,SCREAMING_SNAKE_CASE__=32		,SCREAMING_SNAKE_CASE__=32		,SCREAMING_SNAKE_CASE__=None		,SCREAMING_SNAKE_CASE__="silu"		,SCREAMING_SNAKE_CASE__=20_48		,SCREAMING_SNAKE_CASE__=0.0_2		,SCREAMING_SNAKE_CASE__=1E-6		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=0		,SCREAMING_SNAKE_CASE__=1		,SCREAMING_SNAKE_CASE__=2		,SCREAMING_SNAKE_CASE__=1		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=None		,**SCREAMING_SNAKE_CASE__		,)			->      Union[str, Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  vocab_size
             __SCREAMING_SNAKE_CASE   :int						=  max_position_embeddings
             __SCREAMING_SNAKE_CASE   :List[str]						=  hidden_size
             __SCREAMING_SNAKE_CASE   :Tuple						=  intermediate_size
             __SCREAMING_SNAKE_CASE   :List[str]						=  num_hidden_layers
             __SCREAMING_SNAKE_CASE   :List[Any]						=  num_attention_heads

             # for backward compatibility
             if num_key_value_heads is None:
                   __SCREAMING_SNAKE_CASE   :Optional[int]						=  num_attention_heads

             __SCREAMING_SNAKE_CASE   :str						=  num_key_value_heads
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  hidden_act
             __SCREAMING_SNAKE_CASE   :List[str]						=  initializer_range
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  rms_norm_eps
             __SCREAMING_SNAKE_CASE   :Dict						=  pretraining_tp
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  use_cache
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  rope_scaling
             self._rope_scaling_validation()

             super().__init__(
                 pad_token_id=SCREAMING_SNAKE_CASE__		,bos_token_id=SCREAMING_SNAKE_CASE__		,eos_token_id=SCREAMING_SNAKE_CASE__		,tie_word_embeddings=SCREAMING_SNAKE_CASE__		,**SCREAMING_SNAKE_CASE__		,)




       def     _UpperCamelCase     (						self							)			->      List[Any]:





             """simple docstring"""






             if self.rope_scaling is None:
                   return

             if not isinstance(self.rope_scaling		,SCREAMING_SNAKE_CASE__							) or len(self.rope_scaling							) != 2:
                   raise ValueError(
                       '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
                       f'''got {self.rope_scaling}'''							)
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  self.rope_scaling.get('''type'''		,SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  self.rope_scaling.get('''factor'''		,SCREAMING_SNAKE_CASE__							)
             if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
                   raise ValueError(
                       f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''							)
             if rope_scaling_factor is None or not isinstance(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							) or rope_scaling_factor <= 1.0:
                   raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''							)