




'''simple docstring'''


from transformers.models.cpm.tokenization_cpm import CpmTokenizer
from transformers.testing_utils import custom_tokenizers

from ..xlnet.test_modeling_xlnet import XLNetModelTest






@custom_tokenizers
class 	lowerCAmelCase_  (					UpperCAmelCase_       ):







    '''simple docstring'''

    def 				_snake_case    (   self							:       Optional[int]				,     SCREAMING_SNAKE_CASE_							:       Dict				,     SCREAMING_SNAKE_CASE_							:       Union[str, Any]				,     SCREAMING_SNAKE_CASE_							:       List[Any]				,     SCREAMING_SNAKE_CASE_							:       Dict				,     SCREAMING_SNAKE_CASE_							:       List[Any]  ) ->		int:







         '''simple docstring'''



         return True




    def 				_snake_case    (   self							:       Optional[int]  ) ->		List[str]:







         '''simple docstring'''



         A:		Optional[Any]        =		CpmTokenizer.from_pretrained('''TsinghuaAI/CPM-Generate'''  )
         A:		Optional[int]        =		'''Hugging Face大法好，谁用谁知道。'''
         A:		str        =		'''Hugging Face大法好,谁用谁知道。<unk>'''
         A:		Dict        =		'''▁Hu gg ing ▁ ▂ ▁F ace ▁大法 ▁好 ▁ , ▁谁 ▁用 ▁谁 ▁知 道 ▁ 。'''.split()

         A:		int        =		tokenizer.tokenize(SCREAMING_SNAKE_CASE_  )
         self.assertListEqual(SCREAMING_SNAKE_CASE_				,     SCREAMING_SNAKE_CASE_  )

         A:		Optional[int]        =		tokens + [tokenizer.unk_token]

         A:		str        =		[1_37_89, 1_32_83, 14_21, 8, 10, 11_64, 1_36_08, 1_65_28, 63, 8, 9, 4_40, 1_08, 4_40, 1_21, 90, 8, 12, 0]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_  )				,     SCREAMING_SNAKE_CASE_  )

         A:		Any        =		tokenizer.decode(SCREAMING_SNAKE_CASE_  )
         self.assertEqual(SCREAMING_SNAKE_CASE_				,     SCREAMING_SNAKE_CASE_  )












'''simple docstring'''


from transformers.models.cpm.tokenization_cpm import CpmTokenizer
from transformers.testing_utils import custom_tokenizers

from ..xlnet.test_modeling_xlnet import XLNetModelTest






@custom_tokenizers
class 	lowerCAmelCase_  (					UpperCAmelCase_       ):







  '''simple docstring'''

  def 				_snake_case    (   self							:       Optional[int]				,     SCREAMING_SNAKE_CASE_							:       Dict				,     SCREAMING_SNAKE_CASE_							:       Union[str, Any]				,     SCREAMING_SNAKE_CASE_							:       List[Any]				,     SCREAMING_SNAKE_CASE_							:       Dict				,     SCREAMING_SNAKE_CASE_							:       List[Any]  ) ->		int:







       '''simple docstring'''



       return True




  def 				_snake_case    (   self							:       Optional[int]  ) ->		List[str]:







       '''simple docstring'''



       A:		Optional[Any]        =		CpmTokenizer.from_pretrained('''TsinghuaAI/CPM-Generate'''  )
       A:		Optional[int]        =		'''Hugging Face大法好，谁用谁知道。'''
       A:		str        =		'''Hugging Face大法好,谁用谁知道。<unk>'''
       A:		Dict        =		'''▁Hu gg ing ▁ ▂ ▁F ace ▁大法 ▁好 ▁ , ▁谁 ▁用 ▁谁 ▁知 道 ▁ 。'''.split()

       A:		int        =		tokenizer.tokenize(SCREAMING_SNAKE_CASE_  )
       self.assertListEqual(SCREAMING_SNAKE_CASE_				,     SCREAMING_SNAKE_CASE_  )

       A:		Optional[int]        =		tokens + [tokenizer.unk_token]

       A:		str        =		[1_37_89, 1_32_83, 14_21, 8, 10, 11_64, 1_36_08, 1_65_28, 63, 8, 9, 4_40, 1_08, 4_40, 1_21, 90, 8, 12, 0]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_  )				,     SCREAMING_SNAKE_CASE_  )

       A:		Any        =		tokenizer.decode(SCREAMING_SNAKE_CASE_  )
       self.assertEqual(SCREAMING_SNAKE_CASE_				,     SCREAMING_SNAKE_CASE_  )






