code: import os
  import unittest
  
  from transformers.models.bertweet.tokenization_bertweet import VOCAB_FILES_NAMES, BertweetTokenizer
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  class _lowercase					(       _UpperCAmelCase   ,				unittest.TestCase						):
  
  
  
  
  
  							"""simple docstring"""
  
  
  
  
  							lowerCAmelCase__       =					BertweetTokenizer
  							lowerCAmelCase__       =					False
  
  
  
  							def     _UpperCAmelCase     (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														super().setUp()
  
  														# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
  														_lowercase				=	["""I""", """m""", """V@@""", """R@@""", """r""", """e@@"""]
  														_lowercase				=	dict(zip(UpperCAmelCase     ,   range(len(UpperCAmelCase  )  )  )  )
  														_lowercase				=	["""#version: 0.2""", """a m</w>"""]
  														_lowercase				=	{"""unk_token""": """<unk>"""}
  
  														_lowercase				=	os.path.join(self.tmpdirname     ,   VOCAB_FILES_NAMES["""vocab_file"""]  )
  														_lowercase				=	os.path.join(self.tmpdirname     ,   VOCAB_FILES_NAMES["""merges_file"""]  )
  														with open(self.vocab_file     ,   """w"""     ,   encoding="""utf-8"""  ) as fp:
  																					for token in vocab_tokens:
  																												fp.write(F'''{token} {vocab_tokens[token]}\n'''  )
  														with open(self.merges_file     ,   """w"""     ,   encoding="""utf-8"""  ) as fp:
  																					fp.write("""\n""".join(UpperCAmelCase  )  )
  
  
  
  							def     _UpperCAmelCase     (    self     ,   **UpperCAmelCase  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														kwargs.update(self.special_tokens_map  )
  														return BertweetTokenizer.from_pretrained(self.tmpdirname     ,   **UpperCAmelCase  )
  
  
  
  							def     _UpperCAmelCase     (    self     ,   UpperCAmelCase  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														_lowercase				=	"""I am VinAI Research"""
  														_lowercase				=	"""I <unk> m V<unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"""
  														return input_text, output_text
  
  
  
  							def     _UpperCAmelCase     (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														_lowercase				=	BertweetTokenizer(self.vocab_file     ,   self.merges_file     ,   **self.special_tokens_map  )
  														_lowercase				=	"""I am VinAI Research"""
  														_lowercase				=	"""I a@@ m V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h""".split()
  														_lowercase				=	tokenizer.tokenize(UpperCAmelCase  )
  														self.assertListEqual(UpperCAmelCase     ,   UpperCAmelCase  )
  
  														_lowercase				=	tokens + [tokenizer.unk_token]
  
  														_lowercase				=	[4, 3, 5, 6, 3, 3, 3, 4, 7, 9, 3, 9, 3, 3, 3, 3, 3]
  														self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase  )     ,   UpperCAmelCase  )
  
  
  
style_context: import shutil
  import tempfile
  import unittest
  
  import numpy as np
  import pytest
  
  from transformers.testing_utils import require_vision
  from transformers.utils import is_vision_available
  
  
  if is_vision_available():
  				from PIL import Image
  
  				from transformers import AutoProcessor, BertTokenizer, CLIPImageProcessor, GitProcessor, PreTrainedTokenizerFast
  
  
  @require_vision
  class _lowercase					(       unittest.TestCase						):
  
  
  
  
  
  							"""simple docstring"""
  
  
  
  
  							def     _UpperCAmelCase       (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														_lowercase				=	tempfile.mkdtemp()
  
  														_lowercase				=	CLIPImageProcessor()
  														_lowercase				=	BertTokenizer.from_pretrained(
  														    """hf-internal-testing/tiny-random-BertModel"""     ,   model_input_names=["""input_ids""", """attention_mask"""]  )
  
  														_lowercase				=	GitProcessor(UpperCAmelCase     ,   UpperCAmelCase  )
  
  														processor.save_pretrained(self.tmpdirname  )
  
  
  
  							def     _UpperCAmelCase       (    self     ,   **UpperCAmelCase  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														return AutoProcessor.from_pretrained(self.tmpdirname     ,   **UpperCAmelCase  ).tokenizer
  
  
  
  							def     _UpperCAmelCase       (    self     ,   **UpperCAmelCase  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														return AutoProcessor.from_pretrained(self.tmpdirname     ,   **UpperCAmelCase  ).image_processor
  
  
  
  							def     _UpperCAmelCase       (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														shutil.rmtree(self.tmpdirname  )
  
  
  
  							def     _UpperCAmelCase       (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														_lowercase				=	[np.random.randint(255     ,   size=(3, 30, 400)     ,   dtype=np.uinta  )]
  
  														_lowercase				=	[Image.fromarray(np.moveaxis(UpperCAmelCase     ,   0     ,   -1  )  ) for x in image_inputs]
  
  														return image_inputs
  
  
  
  							def     _UpperCAmelCase       (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														_lowercase				=	GitProcessor(tokenizer=self.get_tokenizer()     ,   image_processor=self.get_image_processor()  )
  														processor.save_pretrained(self.tmpdirname  )
  
  														_lowercase				=	self.get_tokenizer(bos_token="""(BOS)"""     ,   eos_token="""(EOS)"""  )
  														_lowercase				=	self.get_image_processor(do_normalize=UpperCAmelCase     ,   padding_value=1.0  )
  
  														_lowercase				=	GitProcessor.from_pretrained(
  														    self.tmpdirname     ,   bos_token="""(BOS)"""     ,   eos_token="""(EOS)"""     ,   do_normalize=UpperCAmelCase     ,   padding_value=1.0  )
  
  														self.assertEqual(processor.tokenizer.get_vocab()     ,   tokenizer_add_kwargs.get_vocab()  )
  														self.assertIsInstance(processor.tokenizer     ,   UpperCAmelCase  )
  
  														self.assertEqual(processor.image_processor.to_json_string()     ,   image_processor_add_kwargs.to_json_string()  )
  														self.assertIsInstance(processor.image_processor     ,   UpperCAmelCase  )
  
  
  
  							def     _UpperCAmelCase       (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														_lowercase				=	self.get_image_processor()
  														_lowercase				=	self.get_tokenizer()
  
  														_lowercase				=	GitProcessor(tokenizer=UpperCAmelCase     ,   image_processor=UpperCAmelCase  )
  
  														_lowercase				=	self.prepare_image_inputs()
  
  														_lowercase				=	image_processor(UpperCAmelCase     ,   return_tensors="""np"""  )
  														_lowercase				=	processor(images=UpperCAmelCase     ,   return_tensors="""np"""  )
  
  														for key in input_feat_extract.keys():
  																					self.assertAlmostEqual(input_feat_extract[key].sum()     ,   input_processor[key].sum()     ,   delta=1e-2  )
  
  
  
  							def     _UpperCAmelCase       (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														_lowercase				=	self.get_image_processor()
  														_lowercase				=	self.get_tokenizer()
  
  														_lowercase				=	GitProcessor(tokenizer=UpperCAmelCase     ,   image_processor=UpperCAmelCase  )
  
  														_lowercase				=	"""lower newer"""
  
  														_lowercase				=	processor(text=UpperCAmelCase  )
  
  														_lowercase				=	tokenizer(UpperCAmelCase     ,   return_token_type_ids=UpperCAmelCase  )
  
  														for key in encoded_tok.keys():
  																					self.assertListEqual(encoded_tok[key]     ,   encoded_processor[key]  )
  
  
  
  							def     _UpperCAmelCase       (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														_lowercase				=	self.get_image_processor()
  														_lowercase				=	self.get_tokenizer()
  
  														_lowercase				=	GitProcessor(tokenizer=UpperCAmelCase     ,   image_processor=UpperCAmelCase  )
  
  														_lowercase				=	"""lower newer"""
  														_lowercase				=	self.prepare_image_inputs()
  
  														_lowercase				=	processor(text=UpperCAmelCase     ,   images=UpperCAmelCase  )
  
  														self.assertListEqual(list(inputs.keys()  )     ,   ["""input_ids""", """attention_mask""", """pixel_values"""]  )
  
  														# test if it raises when no input is passed
  														with pytest.raises(UpperCAmelCase  ):
  																					processor()
  
  
  
  							def     _UpperCAmelCase       (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														_lowercase				=	self.get_image_processor()
  														_lowercase				=	self.get_tokenizer()
  
  														_lowercase				=	GitProcessor(tokenizer=UpperCAmelCase     ,   image_processor=UpperCAmelCase  )
  
  														_lowercase				=	[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]
  
  														_lowercase				=	processor.batch_decode(UpperCAmelCase  )
  														_lowercase				=	tokenizer.batch_decode(UpperCAmelCase  )
  
  														self.assertListEqual(UpperCAmelCase     ,   UpperCAmelCase  )
  
  
  
  							def     _UpperCAmelCase       (    self  ):
  
  
  
  
  
  
  
  														'''simple docstring'''
  
  
  
  
  
  
  														_lowercase				=	self.get_image_processor()
  														_lowercase				=	self.get_tokenizer()
  
  														_lowercase				=	GitProcessor(tokenizer=UpperCAmelCase     ,   image_processor=UpperCAmelCase  )
  
  														_lowercase				=	"""lower newer"""
  														_lowercase				=	self.prepare_image_inputs()
  
  														_lowercase				=	processor(text=UpperCAmelCase     ,   images=UpperCAmelCase  )
  
  														# For now the processor supports only ['input_ids', 'attention_mask', 'pixel_values']
  														self.assertListEqual(list(inputs.keys()  )     ,   ["""input_ids""", """attention_mask""", """pixel_values"""]  )
  
  
  