code: 
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  import os
  import unittest
  
  from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  
  class 							__snake_case    (				__magic_name__					,					unittest.TestCase       ):
  	__lowerCAmelCase						=     BertweetTokenizer
  	__lowerCAmelCase						=     False
  
  
  
  
  
  	def      _snake_case      (		self		)			->		str:
  		super().setUp()
  
  		# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
  		snake_case__  					=      ['I', 'm', 'V@@', 'R@@', 'r', 'e@@']
  		snake_case__  					=      dict(zip(UpperCamelCase_			,					range(len(UpperCamelCase_		)		)		)		)
  		snake_case__  					=      ['#version: 0.2', 'a m</w>']
  		snake_case__  					=      {'unk_token': '<unk>'}
  
  		snake_case__  					=      os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['vocab_file']		)
  		snake_case__  					=      os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['merges_file']		)
  		with open(self.vocab_file			,					'w'			,					encoding='utf-8'		) as fp:
  			for token in vocab_tokens:
  				fp.write(F'''{token} {vocab_tokens[token]}\n'''		)
  		with open(self.merges_file			,					'w'			,					encoding='utf-8'		) as fp:
  			fp.write('\n'.join(UpperCamelCase_		)		)
  
  
  
  
  
  	def      _snake_case      (		self			,					**UpperCamelCase_		)			->		List[str]:
  		kwargs.update(self.special_tokens_map		)
  		return BertweetTokenizer.from_pretrained(self.tmpdirname			,					**UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_		)			->		str:
  		snake_case__  					=      'I am VinAI Research'
  		snake_case__  					=      'I <unk> m V<unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'
  		return input_text, output_text
  
  
  
  
  
  	def      _snake_case      (		self		)			->		str:
  		snake_case__  					=      BertweetTokenizer(self.vocab_file			,					self.merges_file			,					**self.special_tokens_map		)
  		snake_case__  					=      'I am VinAI Research'
  		snake_case__  					=      'I a@@ m V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'.split()
  		snake_case__  					=      tokenizer.tokenize(UpperCamelCase_		)
  		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)
  
  		snake_case__  					=      tokens + [tokenizer.unk_token]
  
  		snake_case__  					=      [4, 3, 5, 6, 3, 3, 3, 4, 7, 9, 3, 9, 3, 3, 3, 3, 3]
  		self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_		)			,					UpperCamelCase_		)
  
  
  
  
style_context: 
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  from typing import Callable
  
  
  
  
  def 	__lowerCamelCase (  UpperCAmelCase_       )      ->str:
  
  	def wrapper(UpperCAmelCase_       ):
  		snake_case__  					=      function_with_docstring.__doc__
  		return target_function
  
  	return wrapper
  
  
  
  