code: 
  
  
  
  import os
  import unittest
  
  from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  
  class 	lowercase		(				snake_case__				,					unittest.TestCase):
  
  
  
  
  
  
  	"""simple docstring"""
  
  
  	a__   :      Tuple	     =							BertweetTokenizer
  	a__   :      Tuple	     =							False
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Any  )    ->				str:
  							super().setUp()
  
  							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
  							UpperCAmelCase_=     ["""I""", """m""", """V@@""", """R@@""", """r""", """e@@"""]
  							UpperCAmelCase_=     dict(zip(__UpperCAmelCase		,						range(len(__UpperCAmelCase  )  )  )  )
  							UpperCAmelCase_=     ["""#version: 0.2""", """a m</w>"""]
  							UpperCAmelCase_=     {"""unk_token""": """<unk>"""}
  
  							UpperCAmelCase_=     os.path.join(self.tmpdirname		,						VOCAB_FILES_NAMES["""vocab_file"""]  )
  							UpperCAmelCase_=     os.path.join(self.tmpdirname		,						VOCAB_FILES_NAMES["""merges_file"""]  )
  							with open(self.vocab_file		,						"""w"""		,						encoding="""utf-8"""  ) as fp:
  													for token in vocab_tokens:
  																			fp.write(F"""{token} {vocab_tokens[token]}\n"""  )
  							with open(self.merges_file		,						"""w"""		,						encoding="""utf-8"""  ) as fp:
  													fp.write("""\n""".join(__UpperCAmelCase  )  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Union[str, Any]		,						**__UpperCAmelCase			:		Tuple  )    ->				Tuple:
  							kwargs.update(self.special_tokens_map  )
  							return BertweetTokenizer.from_pretrained(self.tmpdirname		,						**__UpperCAmelCase  )
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		str		,						__UpperCAmelCase			:		Tuple  )    ->				int:
  							UpperCAmelCase_=     """I am VinAI Research"""
  							UpperCAmelCase_=     """I <unk> m V<unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"""
  							return input_text, output_text
  
  
  
  
  	def 						_SCREAMING_SNAKE_CASE	(			self			:		Union[str, Any]  )    ->				Dict:
  							UpperCAmelCase_=     BertweetTokenizer(self.vocab_file		,						self.merges_file		,						**self.special_tokens_map  )
  							UpperCAmelCase_=     """I am VinAI Research"""
  							UpperCAmelCase_=     """I a@@ m V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h""".split()
  							UpperCAmelCase_=     tokenizer.tokenize(__UpperCAmelCase  )
  							self.assertListEqual(__UpperCAmelCase		,						__UpperCAmelCase  )
  
  							UpperCAmelCase_=     tokens + [tokenizer.unk_token]
  
  							UpperCAmelCase_=     [4, 3, 5, 6, 3, 3, 3, 4, 7, 9, 3, 9, 3, 3, 3, 3, 3]
  							self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase  )		,						__UpperCAmelCase  )
  
  
  
  
  
style_context: 
  
  
  
  from .prime_check import Test
  
  Test()
  
  
  
  
  