


'''simple docstring'''
import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel
class    lowercase	:

		"""simple docstring"""

		def __init__(		self    ,   UpperCamelCase_    ,   UpperCamelCase_=13    ,   UpperCamelCase_=7    ,   UpperCamelCase_=True    ,   UpperCamelCase_=True    ,   UpperCamelCase_=False    ,   UpperCamelCase_=True    ,   UpperCamelCase_=99    ,   UpperCamelCase_=32    ,   UpperCamelCase_=5    ,   UpperCamelCase_=4    ,   UpperCamelCase_=37    ,   UpperCamelCase_="gelu"    ,   UpperCamelCase_=0.1    ,   UpperCamelCase_=0.1    ,   UpperCamelCase_=512    ,   UpperCamelCase_=16    ,   UpperCamelCase_=2    ,   UpperCamelCase_=0.02    ,   UpperCamelCase_=3    ,   UpperCamelCase_=4    ,   UpperCamelCase_=None    ,   ):

							'''simple docstring'''


							UpperCamelCase__   :Tuple												= parent
							UpperCamelCase__   :Any												= batch_size
							UpperCamelCase__   :Dict												= seq_length
							UpperCamelCase__   :Tuple												= is_training
							UpperCamelCase__   :Dict												= use_input_mask
							UpperCamelCase__   :List[str]												= use_token_type_ids
							UpperCamelCase__   :Dict												= use_labels
							UpperCamelCase__   :Optional[int]												= vocab_size
							UpperCamelCase__   :Dict												= hidden_size
							UpperCamelCase__   :Tuple												= num_hidden_layers
							UpperCamelCase__   :Any												= num_attention_heads
							UpperCamelCase__   :Tuple												= intermediate_size
							UpperCamelCase__   :Any												= hidden_act
							UpperCamelCase__   :Optional[Any]												= hidden_dropout_prob
							UpperCamelCase__   :Union[str, Any]												= attention_probs_dropout_prob
							UpperCamelCase__   :Optional[Any]												= max_position_embeddings
							UpperCamelCase__   :Optional[Any]												= type_vocab_size
							UpperCamelCase__   :Optional[Any]												= type_sequence_label_size
							UpperCamelCase__   :Dict												= initializer_range
							UpperCamelCase__   :Union[str, Any]												= num_labels
							UpperCamelCase__   :Dict												= num_choices
							UpperCamelCase__   :List[Any]												= scope




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :Any												= ids_tensor([self.batch_size, self.seq_length]    ,   self.vocab_size							)

							UpperCamelCase__   :Tuple												= None
							if self.use_input_mask:
												UpperCamelCase__   :Tuple												= random_attention_mask([self.batch_size, self.seq_length]							)

							UpperCamelCase__   :Optional[Any]												= None
							if self.use_token_type_ids:
												UpperCamelCase__   :List[Any]												= ids_tensor([self.batch_size, self.seq_length]    ,   self.type_vocab_size							)

							UpperCamelCase__   :int												= None
							UpperCamelCase__   :Optional[int]												= None
							UpperCamelCase__   :List[str]												= None
							if self.use_labels:
												UpperCamelCase__   :Optional[int]												= ids_tensor([self.batch_size]    ,   self.type_sequence_label_size							)
												UpperCamelCase__   :str												= ids_tensor([self.batch_size, self.seq_length]    ,   self.num_labels							)
												UpperCamelCase__   :Tuple												= ids_tensor([self.batch_size]    ,   self.num_choices							)

							UpperCamelCase__   :Union[str, Any]												= self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							return OpenLlamaConfig(
							    vocab_size=self.vocab_size    ,   hidden_size=self.hidden_size    ,   num_hidden_layers=self.num_hidden_layers    ,   num_attention_heads=self.num_attention_heads    ,   intermediate_size=self.intermediate_size    ,   hidden_act=self.hidden_act    ,   hidden_dropout_prob=self.hidden_dropout_prob    ,   attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,   max_position_embeddings=self.max_position_embeddings    ,   type_vocab_size=self.type_vocab_size    ,   is_decoder=__snake_case    ,   initializer_range=self.initializer_range    ,   use_stable_embedding=__snake_case    ,   )




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__   :str												= OpenLlamaModel(config=__snake_case							)
							model.to(__snake_case							)
							model.eval()
							UpperCamelCase__   :Optional[int]												= model(__snake_case    ,   attention_mask=__snake_case							)
							UpperCamelCase__   :Union[str, Any]												= model(__snake_case							)
							self.parent.assertEqual(result.last_hidden_state.shape    ,   (self.batch_size, self.seq_length, self.hidden_size)							)




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   ):

							'''simple docstring'''


							UpperCamelCase__   :Tuple												= True
							UpperCamelCase__   :str												= OpenLlamaModel(__snake_case							)
							model.to(__snake_case							)
							model.eval()
							UpperCamelCase__   :Union[str, Any]												= model(
							    __snake_case    ,   attention_mask=__snake_case    ,   encoder_hidden_states=__snake_case    ,   encoder_attention_mask=__snake_case    ,   )
							UpperCamelCase__   :Dict												= model(
							    __snake_case    ,   attention_mask=__snake_case    ,   encoder_hidden_states=__snake_case    ,   )
							UpperCamelCase__   :int												= model(__snake_case    ,   attention_mask=__snake_case							)
							self.parent.assertEqual(result.last_hidden_state.shape    ,   (self.batch_size, self.seq_length, self.hidden_size)							)




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   ):

							'''simple docstring'''


							UpperCamelCase__   :List[Any]												= OpenLlamaForCausalLM(config=__snake_case							)
							model.to(__snake_case							)
							model.eval()
							UpperCamelCase__   :List[Any]												= model(__snake_case    ,   attention_mask=__snake_case    ,   labels=__snake_case							)
							self.parent.assertEqual(result.logits.shape    ,   (self.batch_size, self.seq_length, self.vocab_size)							)




		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   UpperCamelCase_    ,   ):

							'''simple docstring'''


							UpperCamelCase__   :str												= True
							UpperCamelCase__   :List[str]												= True
							UpperCamelCase__   :Optional[int]												= OpenLlamaForCausalLM(config=__snake_case							)
							model.to(__snake_case							)
							model.eval()

							# first forward pass
							UpperCamelCase__   :List[str]												= model(
							    __snake_case    ,   attention_mask=__snake_case    ,   encoder_hidden_states=__snake_case    ,   encoder_attention_mask=__snake_case    ,   use_cache=__snake_case    ,   )
							UpperCamelCase__   :List[Any]												= outputs.past_key_values

							# create hypothetical multiple next token and extent to next_input_ids
							UpperCamelCase__   :List[Any]												= ids_tensor((self.batch_size, 3)    ,   config.vocab_size							)
							UpperCamelCase__   :Optional[int]												= ids_tensor((self.batch_size, 3)    ,   vocab_size=2							)

							# append to next input_ids and
							UpperCamelCase__   :int												= torch.cat([input_ids, next_tokens]    ,   dim=-1							)
							UpperCamelCase__   :List[Any]												= torch.cat([input_mask, next_mask]    ,   dim=-1							)

							UpperCamelCase__   :Optional[int]												= model(
							    __snake_case    ,   attention_mask=__snake_case    ,   encoder_hidden_states=__snake_case    ,   encoder_attention_mask=__snake_case    ,   output_hidden_states=__snake_case    ,   )['''hidden_states'''][0]
							UpperCamelCase__   :List[Any]												= model(
							    __snake_case    ,   attention_mask=__snake_case    ,   encoder_hidden_states=__snake_case    ,   encoder_attention_mask=__snake_case    ,   past_key_values=__snake_case    ,   output_hidden_states=__snake_case    ,   )['''hidden_states'''][0]

							# select random slice
							UpperCamelCase__   :Any												= ids_tensor((1,)    ,   output_from_past.shape[-1]							).item()
							UpperCamelCase__   :int												= output_from_no_past[:, -3:, random_slice_idx].detach()
							UpperCamelCase__   :Dict												= output_from_past[:, :, random_slice_idx].detach()

							self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]							)

							# test that outputs are equal for slice
							self.parent.assertTrue(torch.allclose(__snake_case    ,   __snake_case    ,   atol=1e-3							)							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :List[str]												= self.prepare_config_and_inputs()
							(
							    (
							    UpperCamelCase__    
							)					,				(
							    UpperCamelCase__    
							)					,				(
							    UpperCamelCase__    
							)					,				(
							    UpperCamelCase__    
							)					,				(
							    UpperCamelCase__    
							)					,				(
							    UpperCamelCase__    
							)					,				(
							    UpperCamelCase__    
							)					,				    
							)   :Optional[Any]												= config_and_inputs
							UpperCamelCase__   :List[str]												= {'''input_ids''': input_ids, '''attention_mask''': input_mask}
							return config, inputs_dict



@require_torch
class    lowercase	(							lowerCAmelCase_	,		lowerCAmelCase_	,		lowerCAmelCase_	,		unittest.TestCase					):

		"""simple docstring"""

		_a     =				(
		    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
		)
		_a     =				(OpenLlamaForCausalLM,) if is_torch_available() else ()
		_a     =				(
		    {
		        'feature-extraction': OpenLlamaModel,
		        'text-classification': OpenLlamaForSequenceClassification,
		        'text-generation': OpenLlamaForCausalLM,
		        'zero-shot': OpenLlamaForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		_a     =				False
		_a     =				False




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :str												= OpenLlamaModelTester(self							)
							UpperCamelCase__   :Tuple												= ConfigTester(self    ,   config_class=__snake_case    ,   hidden_size=37							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							self.config_tester.run_common_tests()




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :Union[str, Any]												= self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*__snake_case							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__   :int												= self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
												UpperCamelCase__   :List[str]												= type
												self.model_tester.create_and_check_model(*__snake_case							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__					,				UpperCamelCase__   :Tuple												= self.model_tester.prepare_config_and_inputs_for_common()
							UpperCamelCase__   :str												= 3
							UpperCamelCase__   :Optional[Any]												= input_dict['''input_ids''']
							UpperCamelCase__   :int												= input_ids.ne(1							).to(__snake_case							)
							UpperCamelCase__   :Optional[Any]												= ids_tensor([self.model_tester.batch_size]    ,   self.model_tester.type_sequence_label_size							)
							UpperCamelCase__   :Union[str, Any]												= OpenLlamaForSequenceClassification(__snake_case							)
							model.to(__snake_case							)
							model.eval()
							UpperCamelCase__   :Optional[Any]												= model(__snake_case    ,   attention_mask=__snake_case    ,   labels=__snake_case							)
							self.assertEqual(result.logits.shape    ,   (self.model_tester.batch_size, self.model_tester.num_labels)							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__					,				UpperCamelCase__   :List[Any]												= self.model_tester.prepare_config_and_inputs_for_common()
							UpperCamelCase__   :Dict												= 3
							UpperCamelCase__   :Any												= '''single_label_classification'''
							UpperCamelCase__   :Dict												= input_dict['''input_ids''']
							UpperCamelCase__   :Dict												= input_ids.ne(1							).to(__snake_case							)
							UpperCamelCase__   :Optional[Any]												= ids_tensor([self.model_tester.batch_size]    ,   self.model_tester.type_sequence_label_size							)
							UpperCamelCase__   :Any												= OpenLlamaForSequenceClassification(__snake_case							)
							model.to(__snake_case							)
							model.eval()
							UpperCamelCase__   :Optional[int]												= model(__snake_case    ,   attention_mask=__snake_case    ,   labels=__snake_case							)
							self.assertEqual(result.logits.shape    ,   (self.model_tester.batch_size, self.model_tester.num_labels)							)




		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							UpperCamelCase__					,				UpperCamelCase__   :List[Any]												= self.model_tester.prepare_config_and_inputs_for_common()
							UpperCamelCase__   :Dict												= 3
							UpperCamelCase__   :Tuple												= '''multi_label_classification'''
							UpperCamelCase__   :Optional[int]												= input_dict['''input_ids''']
							UpperCamelCase__   :int												= input_ids.ne(1							).to(__snake_case							)
							UpperCamelCase__   :Tuple												= ids_tensor(
							    [self.model_tester.batch_size, config.num_labels]    ,   self.model_tester.type_sequence_label_size							).to(torch.float							)
							UpperCamelCase__   :Dict												= OpenLlamaForSequenceClassification(__snake_case							)
							model.to(__snake_case							)
							model.eval()
							UpperCamelCase__   :Union[str, Any]												= model(__snake_case    ,   attention_mask=__snake_case    ,   labels=__snake_case							)
							self.assertEqual(result.logits.shape    ,   (self.model_tester.batch_size, self.model_tester.num_labels)							)




		@unittest.skip('''Open-Llama buffers include complex numbers, which breaks this test'''							)
		def     lowerCAmelCase__				(		self							):

							'''simple docstring'''


							pass




		@parameterized.expand([('''linear''',), ('''dynamic''',)]							)
		def     lowerCAmelCase__				(		self    ,   UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__					,				UpperCamelCase__   :int												= self.model_tester.prepare_config_and_inputs_for_common()
							UpperCamelCase__   :Dict												= ids_tensor([1, 10]    ,   config.vocab_size							)
							UpperCamelCase__   :str												= ids_tensor([1, int(config.max_position_embeddings * 1.5							)]    ,   config.vocab_size							)

							set_seed(42							)  # Fixed seed at init time so the two models get the same random weights
							UpperCamelCase__   :Dict												= OpenLlamaModel(__snake_case							)
							original_model.to(__snake_case							)
							original_model.eval()
							UpperCamelCase__   :Optional[int]												= original_model(__snake_case							).last_hidden_state
							UpperCamelCase__   :Optional[Any]												= original_model(__snake_case							).last_hidden_state

							set_seed(42							)  # Fixed seed at init time so the two models get the same random weights
							UpperCamelCase__   :Any												= {'''type''': scaling_type, '''factor''': 10.0}
							UpperCamelCase__   :Any												= OpenLlamaModel(__snake_case							)
							scaled_model.to(__snake_case							)
							scaled_model.eval()
							UpperCamelCase__   :List[str]												= scaled_model(__snake_case							).last_hidden_state
							UpperCamelCase__   :str												= scaled_model(__snake_case							).last_hidden_state

							# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
							# maximum sequence length, so the outputs for the short input should match.
							if scaling_type == "dynamic":
												self.assertTrue(torch.allclose(__snake_case    ,   __snake_case    ,   atol=1e-5							)							)
							else:
												self.assertFalse(torch.allclose(__snake_case    ,   __snake_case    ,   atol=1e-5							)							)

							# The output should be different for long inputs
							self.assertFalse(torch.allclose(__snake_case    ,   __snake_case    ,   atol=1e-5							)							)


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 	__magic_name__    (			lowerCAmelCase_  ,    unittest.TestCase      ):
			SCREAMING_SNAKE_CASE							   =						RoCBertTokenizer
			SCREAMING_SNAKE_CASE							   =						None
			SCREAMING_SNAKE_CASE							   =						False
			SCREAMING_SNAKE_CASE							   =						True
			SCREAMING_SNAKE_CASE							   =						filter_non_english





			def 		__magic_name__    (		self					)   -> Tuple:




						'''simple docstring'''
						super().setUp()

						__a        =['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', '你', '好', '是', '谁', 'a', 'b', 'c', 'd']
						__a        ={}
						__a        ={}
						for i, value in enumerate(__snake_case					):
									__a        =i
									__a        =i
						__a        =os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES['vocab_file']					)
						__a        =os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES['word_shape_file']					)
						__a        =os.path.join(self.tmpdirname					,  VOCAB_FILES_NAMES['word_pronunciation_file']					)
						with open(self.vocab_file					,  'w'					,  encoding='utf-8'					) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)
						with open(self.word_shape_file					,  'w'					,  encoding='utf-8'					) as word_shape_writer:
									json.dump(__snake_case					,  __snake_case					,  ensure_ascii=__snake_case					)
						with open(self.word_pronunciation_file					,  'w'					,  encoding='utf-8'					) as word_pronunciation_writer:
									json.dump(__snake_case					,  __snake_case					,  ensure_ascii=__snake_case					)





			def 		__magic_name__    (		self					)   -> Any:




						'''simple docstring'''
						__a        =self.tokenizer_class(self.vocab_file					,  self.word_shape_file					,  self.word_pronunciation_file					)

						__a        =tokenizer.tokenize('你好[SEP]你是谁'					)
						self.assertListEqual(__snake_case					,  ['你', '好', '[SEP]', '你', '是', '谁']					)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case					)					,  [5, 6, 2, 5, 7, 8]					)
						self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__snake_case					)					,  [5, 6, 2, 5, 7, 8]					)
						self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__snake_case					)					,  [5, 6, 2, 5, 7, 8]					)





			def 		__magic_name__    (		self					)   -> Any:




						'''simple docstring'''
						__a        =RoCBertBasicTokenizer()

						self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'					)					,  ['ah', '\u535A', '\u63A8', 'zz']					)





			def 		__magic_name__    (		self					)   -> Optional[Any]:




						'''simple docstring'''
						__a        =RoCBertBasicTokenizer(do_lower_case=__snake_case					)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)					,  ['hello', '!', 'how', 'are', 'you', '?']					)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)					,  ['hello']					)





			def 		__magic_name__    (		self					)   -> Optional[Any]:




						'''simple docstring'''
						__a        =RoCBertBasicTokenizer(do_lower_case=__snake_case					,  strip_accents=__snake_case					)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)					,  ['hällo', '!', 'how', 'are', 'you', '?']					)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)					,  ['h\u00E9llo']					)





			def 		__magic_name__    (		self					)   -> Optional[Any]:




						'''simple docstring'''
						__a        =RoCBertBasicTokenizer(do_lower_case=__snake_case					,  strip_accents=__snake_case					)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)					,  ['hallo', '!', 'how', 'are', 'you', '?']					)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)					,  ['hello']					)





			def 		__magic_name__    (		self					)   -> int:




						'''simple docstring'''
						__a        =RoCBertBasicTokenizer(do_lower_case=__snake_case					)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)					,  ['hallo', '!', 'how', 'are', 'you', '?']					)
						self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)					,  ['hello']					)





			def 		__magic_name__    (		self					)   -> Dict:




						'''simple docstring'''
						__a        =RoCBertBasicTokenizer(do_lower_case=__snake_case					)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)					,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)





			def 		__magic_name__    (		self					)   -> Union[str, Any]:




						'''simple docstring'''
						__a        =RoCBertBasicTokenizer(do_lower_case=__snake_case					,  strip_accents=__snake_case					)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)					,  ['HäLLo', '!', 'how', 'Are', 'yoU', '?']					)





			def 		__magic_name__    (		self					)   -> int:




						'''simple docstring'''
						__a        =RoCBertBasicTokenizer(do_lower_case=__snake_case					,  strip_accents=__snake_case					)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)					,  ['HaLLo', '!', 'how', 'Are', 'yoU', '?']					)





			def 		__magic_name__    (		self					)   -> Tuple:




						'''simple docstring'''
						__a        =RoCBertBasicTokenizer(do_lower_case=__snake_case					,  never_split=['[UNK]']					)

						self.assertListEqual(
						    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'					)					,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']					)





			def 		__magic_name__    (		self					)   -> int:




						'''simple docstring'''
						__a        =['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

						__a        ={}
						for i, token in enumerate(__snake_case					):
									__a        =i
						__a        =RoCBertWordpieceTokenizer(vocab=__snake_case					,  unk_token='[UNK]'					)

						self.assertListEqual(tokenizer.tokenize(''					)					,  []					)

						self.assertListEqual(tokenizer.tokenize('unwanted running'					)					,  ['un', '##want', '##ed', 'runn', '##ing']					)

						self.assertListEqual(tokenizer.tokenize('unwantedX running'					)					,  ['[UNK]', 'runn', '##ing']					)





			def 		__magic_name__    (		self					)   -> str:




						'''simple docstring'''
						self.assertTrue(_is_whitespace(' '					)					)
						self.assertTrue(_is_whitespace('\t'					)					)
						self.assertTrue(_is_whitespace('\r'					)					)
						self.assertTrue(_is_whitespace('\n'					)					)
						self.assertTrue(_is_whitespace('\u00A0'					)					)

						self.assertFalse(_is_whitespace('A'					)					)
						self.assertFalse(_is_whitespace('-'					)					)





			def 		__magic_name__    (		self					)   -> Optional[int]:




						'''simple docstring'''
						self.assertTrue(_is_control('\u0005'					)					)

						self.assertFalse(_is_control('A'					)					)
						self.assertFalse(_is_control(' '					)					)
						self.assertFalse(_is_control('\t'					)					)
						self.assertFalse(_is_control('\r'					)					)





			def 		__magic_name__    (		self					)   -> Optional[Any]:




						'''simple docstring'''
						self.assertTrue(_is_punctuation('-'					)					)
						self.assertTrue(_is_punctuation('$'					)					)
						self.assertTrue(_is_punctuation('`'					)					)
						self.assertTrue(_is_punctuation('.'					)					)

						self.assertFalse(_is_punctuation('A'					)					)
						self.assertFalse(_is_punctuation(' '					)					)





			def 		__magic_name__    (		self					)   -> Any:




						'''simple docstring'''
						__a        =self.get_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(__snake_case					) for t in ['Test', '\xad', 'test']]					,  [['[UNK]'], [], ['[UNK]']]					)

						if self.test_rust_tokenizer:
									__a        =self.get_rust_tokenizer()
									self.assertListEqual(
									    [rust_tokenizer.tokenize(__snake_case					) for t in ['Test', '\xad', 'test']]					,  [['[UNK]'], [], ['[UNK]']]					)





			def 		__magic_name__    (		self					)   -> Any:




						'''simple docstring'''
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
												__a        =self.rust_tokenizer_class.from_pretrained(__snake_case					,  **__snake_case					)

												__a        =f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
												__a        =tokenizer_r.encode_plus(
												    __snake_case					,  return_attention_mask=__snake_case					,  return_token_type_ids=__snake_case					,  return_offsets_mapping=__snake_case					,  add_special_tokens=__snake_case					,  )

												__a        =tokenizer_r.do_lower_case if hasattr(__snake_case					,  'do_lower_case'					) else False
												__a        =(
												    [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), 'A'),
												        ((1, 2), ','),
												        ((3, 5), 'na'),
												        ((5, 6), '##ï'),
												        ((6, 8), '##ve'),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), 'Allen'),
												        ((21, 23), '##NL'),
												        ((23, 24), '##P'),
												        ((25, 33), 'sentence'),
												        ((33, 34), '.'),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												    if not do_lower_case
												    else [
												        ((0, 0), tokenizer_r.cls_token),
												        ((0, 1), 'a'),
												        ((1, 2), ','),
												        ((3, 8), 'naive'),
												        ((9, 15), tokenizer_r.mask_token),
												        ((16, 21), 'allen'),
												        ((21, 23), '##nl'),
												        ((23, 24), '##p'),
												        ((25, 33), 'sentence'),
												        ((33, 34), '.'),
												        ((0, 0), tokenizer_r.sep_token),
												    ]
												)

												self.assertEqual(
												    [e[1] for e in expected_results]					,  tokenizer_r.convert_ids_to_tokens(tokens['input_ids']					)					)
												self.assertEqual([e[0] for e in expected_results]					,  tokens['offset_mapping']					)





			def 		__magic_name__    (		self					)   -> List[Any]:




						'''simple docstring'''
						__a        =['的', '人', '有']
						__a        =''.join(__snake_case					)
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
												__a        =True
												__a        =self.tokenizer_class.from_pretrained(__snake_case					,  **__snake_case					)
												__a        =self.rust_tokenizer_class.from_pretrained(__snake_case					,  **__snake_case					)

												__a        =tokenizer_p.encode(__snake_case					,  add_special_tokens=__snake_case					)
												__a        =tokenizer_r.encode(__snake_case					,  add_special_tokens=__snake_case					)

												__a        =tokenizer_r.convert_ids_to_tokens(__snake_case					)
												__a        =tokenizer_p.convert_ids_to_tokens(__snake_case					)

												# it is expected that each Chinese character is not preceded by "##"
												self.assertListEqual(__snake_case					,  __snake_case					)
												self.assertListEqual(__snake_case					,  __snake_case					)

												__a        =False
												__a        =self.rust_tokenizer_class.from_pretrained(__snake_case					,  **__snake_case					)
												__a        =self.tokenizer_class.from_pretrained(__snake_case					,  **__snake_case					)

												__a        =tokenizer_r.encode(__snake_case					,  add_special_tokens=__snake_case					)
												__a        =tokenizer_p.encode(__snake_case					,  add_special_tokens=__snake_case					)

												__a        =tokenizer_r.convert_ids_to_tokens(__snake_case					)
												__a        =tokenizer_p.convert_ids_to_tokens(__snake_case					)

												# it is expected that only the first Chinese character is not preceded by "##".
												__a        =[
												    f'##{token}' if idx != 0 else token for idx, token in enumerate(__snake_case					)
												]
												self.assertListEqual(__snake_case					,  __snake_case					)
												self.assertListEqual(__snake_case					,  __snake_case					)





			@slow
			def 		__magic_name__    (		self					)   -> Optional[int]:




						'''simple docstring'''
						__a        =self.tokenizer_class(self.vocab_file					,  self.word_shape_file					,  self.word_pronunciation_file					)

						__a        =tokenizer.encode('你好'					,  add_special_tokens=__snake_case					)
						__a        =tokenizer.encode('你是谁'					,  add_special_tokens=__snake_case					)

						__a        =tokenizer.build_inputs_with_special_tokens(__snake_case					)
						__a        =tokenizer.build_inputs_with_special_tokens(__snake_case					,  __snake_case					)

						assert encoded_sentence == [1] + text + [2]
						assert encoded_pair == [1] + text + [2] + text_a + [2]





			def 		__magic_name__    (		self					)   -> List[str]:




						'''simple docstring'''
						__a        =self.get_tokenizers(do_lower_case=__snake_case					)
						for tokenizer in tokenizers:
									with self.subTest(f'{tokenizer.__class__.__name__}'					):
												__a        ='你好，你是谁'
												__a        =tokenizer.tokenize(__snake_case					)
												__a        =tokenizer.convert_tokens_to_ids(__snake_case					)
												__a        =tokenizer.convert_tokens_to_shape_ids(__snake_case					)
												__a        =tokenizer.convert_tokens_to_pronunciation_ids(__snake_case					)
												__a        =tokenizer.prepare_for_model(
												    __snake_case					,  __snake_case					,  __snake_case					,  add_special_tokens=__snake_case					)

												__a        =tokenizer.encode_plus(__snake_case					,  add_special_tokens=__snake_case					)

												self.assertEqual(__snake_case					,  __snake_case					)






