






'''simple docstring'''
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers.testing_utils import require_vision
from transformers.utils import is_vision_available


if is_vision_available():
  from PIL import Image

  from transformers import (
      AutoProcessor,
      BertTokenizerFast,
      BlipImageProcessor,
      GPTaTokenizer,
      InstructBlipProcessor,
      PreTrainedTokenizerFast,
  )






@require_vision
class    __SCREAMING_SNAKE_CASE					(	unittest.TestCase ):




 def    __magic_name__		(					self		:							Union[str, Any]					)		->  Any:
       SCREAMING_SNAKE_CASE__   : List[Any]													=tempfile.mkdtemp()

       SCREAMING_SNAKE_CASE__   : int													=BlipImageProcessor()
       SCREAMING_SNAKE_CASE__   : Any													=GPTaTokenizer.from_pretrained('''hf-internal-testing/tiny-random-GPT2Model'''					)
       SCREAMING_SNAKE_CASE__   : Union[str, Any]													=BertTokenizerFast.from_pretrained('''hf-internal-testing/tiny-random-bert'''					)

       SCREAMING_SNAKE_CASE__   : Any													=InstructBlipProcessor(UpperCAmelCase_    ,					UpperCAmelCase_    ,					UpperCAmelCase_					)

       processor.save_pretrained(self.tmpdirname					)




 def    __magic_name__		(					self		:							Optional[int]    ,					**__lowercase		:							int					)		->  str:
       return AutoProcessor.from_pretrained(self.tmpdirname    ,					**UpperCAmelCase_					).tokenizer




 def    __magic_name__		(					self		:							Optional[int]    ,					**__lowercase		:							Tuple					)		->  Optional[int]:
       return AutoProcessor.from_pretrained(self.tmpdirname    ,					**UpperCAmelCase_					).image_processor




 def    __magic_name__		(					self		:							Union[str, Any]    ,					**__lowercase		:							Any					)		->  int:
       return AutoProcessor.from_pretrained(self.tmpdirname    ,					**UpperCAmelCase_					).qformer_tokenizer




 def    __magic_name__		(					self		:							Tuple					)		->  Tuple:
       shutil.rmtree(self.tmpdirname					)




 def    __magic_name__		(					self		:							Union[str, Any]					)		->  Optional[int]:
       SCREAMING_SNAKE_CASE__   : List[str]													=[np.random.randint(2_55    ,					size=(3, 30, 4_00)    ,					dtype=np.uinta					)]

       SCREAMING_SNAKE_CASE__   : List[str]													=[Image.fromarray(np.moveaxis(UpperCAmelCase_    ,					0    ,					-1					)					) for x in image_inputs]

       return image_inputs




 def    __magic_name__		(					self		:							Tuple					)		->  Dict:
       SCREAMING_SNAKE_CASE__   : List[Any]													=InstructBlipProcessor(
           tokenizer=self.get_tokenizer()    ,					image_processor=self.get_image_processor()    ,					qformer_tokenizer=self.get_qformer_tokenizer()    ,					)
       processor.save_pretrained(self.tmpdirname					)

       SCREAMING_SNAKE_CASE__   : Dict													=self.get_tokenizer(bos_token='''(BOS)'''    ,					eos_token='''(EOS)'''					)
       SCREAMING_SNAKE_CASE__   : Dict													=self.get_image_processor(do_normalize=UpperCAmelCase_    ,					padding_value=1.0					)

       SCREAMING_SNAKE_CASE__   : Optional[Any]													=InstructBlipProcessor.from_pretrained(
           self.tmpdirname    ,					bos_token='''(BOS)'''    ,					eos_token='''(EOS)'''    ,					do_normalize=UpperCAmelCase_    ,					padding_value=1.0					)

       self.assertEqual(processor.tokenizer.get_vocab()    ,					tokenizer_add_kwargs.get_vocab()					)
       self.assertIsInstance(processor.tokenizer    ,					UpperCAmelCase_					)

       self.assertEqual(processor.image_processor.to_json_string()    ,					image_processor_add_kwargs.to_json_string()					)
       self.assertIsInstance(processor.image_processor    ,					UpperCAmelCase_					)
       self.assertIsInstance(processor.qformer_tokenizer    ,					UpperCAmelCase_					)




 def    __magic_name__		(					self		:							Dict					)		->  Optional[Any]:
       SCREAMING_SNAKE_CASE__   : List[Any]													=self.get_image_processor()
       SCREAMING_SNAKE_CASE__   : int													=self.get_tokenizer()
       SCREAMING_SNAKE_CASE__   : Optional[int]													=self.get_qformer_tokenizer()

       SCREAMING_SNAKE_CASE__   : Optional[Any]													=InstructBlipProcessor(
           tokenizer=UpperCAmelCase_    ,					image_processor=UpperCAmelCase_    ,					qformer_tokenizer=UpperCAmelCase_					)

       SCREAMING_SNAKE_CASE__   : Dict													=self.prepare_image_inputs()

       SCREAMING_SNAKE_CASE__   : Tuple													=image_processor(UpperCAmelCase_    ,					return_tensors='''np'''					)
       SCREAMING_SNAKE_CASE__   : Any													=processor(images=UpperCAmelCase_    ,					return_tensors='''np'''					)

       for key in input_feat_extract.keys():
             self.assertAlmostEqual(input_feat_extract[key].sum()    ,					input_processor[key].sum()    ,					delta=1e-2					)




 def    __magic_name__		(					self		:							Dict					)		->  int:
       SCREAMING_SNAKE_CASE__   : Any													=self.get_image_processor()
       SCREAMING_SNAKE_CASE__   : str													=self.get_tokenizer()
       SCREAMING_SNAKE_CASE__   : Optional[int]													=self.get_qformer_tokenizer()

       SCREAMING_SNAKE_CASE__   : Union[str, Any]													=InstructBlipProcessor(
           tokenizer=UpperCAmelCase_    ,					image_processor=UpperCAmelCase_    ,					qformer_tokenizer=UpperCAmelCase_					)

       SCREAMING_SNAKE_CASE__   : Optional[int]													='''lower newer'''

       SCREAMING_SNAKE_CASE__   : Optional[Any]													=processor(text=UpperCAmelCase_					)

       SCREAMING_SNAKE_CASE__   : Dict													=tokenizer(UpperCAmelCase_    ,					return_token_type_ids=UpperCAmelCase_					)
       SCREAMING_SNAKE_CASE__   : int													=qformer_tokenizer(UpperCAmelCase_    ,					return_token_type_ids=UpperCAmelCase_					)

       for key in encoded_tokens.keys():
             self.assertListEqual(encoded_tokens[key]    ,					encoded_processor[key]					)

       for key in encoded_tokens_qformer.keys():
             self.assertListEqual(encoded_tokens_qformer[key]    ,					encoded_processor['''qformer_''' + key]					)




 def    __magic_name__		(					self		:							Union[str, Any]					)		->  Optional[Any]:
       SCREAMING_SNAKE_CASE__   : Optional[Any]													=self.get_image_processor()
       SCREAMING_SNAKE_CASE__   : str													=self.get_tokenizer()
       SCREAMING_SNAKE_CASE__   : str													=self.get_qformer_tokenizer()

       SCREAMING_SNAKE_CASE__   : Dict													=InstructBlipProcessor(
           tokenizer=UpperCAmelCase_    ,					image_processor=UpperCAmelCase_    ,					qformer_tokenizer=UpperCAmelCase_					)

       SCREAMING_SNAKE_CASE__   : Union[str, Any]													='''lower newer'''
       SCREAMING_SNAKE_CASE__   : List[str]													=self.prepare_image_inputs()

       SCREAMING_SNAKE_CASE__   : Optional[Any]													=processor(text=UpperCAmelCase_    ,					images=UpperCAmelCase_					)

       self.assertListEqual(
           list(inputs.keys()					)    ,					['''input_ids''', '''attention_mask''', '''qformer_input_ids''', '''qformer_attention_mask''', '''pixel_values''']    ,					)

       # test if it raises when no input is passed
       with pytest.raises(UpperCAmelCase_					):
             processor()




 def    __magic_name__		(					self		:							str					)		->  str:
       SCREAMING_SNAKE_CASE__   : Optional[int]													=self.get_image_processor()
       SCREAMING_SNAKE_CASE__   : List[Any]													=self.get_tokenizer()
       SCREAMING_SNAKE_CASE__   : Any													=self.get_qformer_tokenizer()

       SCREAMING_SNAKE_CASE__   : Any													=InstructBlipProcessor(
           tokenizer=UpperCAmelCase_    ,					image_processor=UpperCAmelCase_    ,					qformer_tokenizer=UpperCAmelCase_					)

       SCREAMING_SNAKE_CASE__   : str													=[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

       SCREAMING_SNAKE_CASE__   : Optional[int]													=processor.batch_decode(UpperCAmelCase_					)
       SCREAMING_SNAKE_CASE__   : Union[str, Any]													=tokenizer.batch_decode(UpperCAmelCase_					)

       self.assertListEqual(UpperCAmelCase_    ,					UpperCAmelCase_					)




 def    __magic_name__		(					self		:							Dict					)		->  List[Any]:
       SCREAMING_SNAKE_CASE__   : Optional[int]													=self.get_image_processor()
       SCREAMING_SNAKE_CASE__   : Dict													=self.get_tokenizer()
       SCREAMING_SNAKE_CASE__   : Optional[int]													=self.get_qformer_tokenizer()

       SCREAMING_SNAKE_CASE__   : int													=InstructBlipProcessor(
           tokenizer=UpperCAmelCase_    ,					image_processor=UpperCAmelCase_    ,					qformer_tokenizer=UpperCAmelCase_					)

       SCREAMING_SNAKE_CASE__   : Optional[int]													='''lower newer'''
       SCREAMING_SNAKE_CASE__   : Dict													=self.prepare_image_inputs()

       SCREAMING_SNAKE_CASE__   : Optional[Any]													=processor(text=UpperCAmelCase_    ,					images=UpperCAmelCase_					)

       self.assertListEqual(
           list(inputs.keys()					)    ,					['''input_ids''', '''attention_mask''', '''qformer_input_ids''', '''qformer_attention_mask''', '''pixel_values''']    ,					)

import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 						lowercase__  (						_UpperCAmelCase     ,       unittest.TestCase  ):
	A__    :   Any				       =GPTSanJapaneseTokenizer
	A__    :   str				       =False
	A__    :   int				       ={"""do_clean_text""": False, """add_prefix_space""": False}
	def      A_				(			self							:   Any							):
						super().setUp()

						# fmt: off
						SCREAMING_SNAKE_CASE__													= ['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
						# fmt: on
						SCREAMING_SNAKE_CASE__													= {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
						SCREAMING_SNAKE_CASE__													= {'unk_token': '<unk>'}

						SCREAMING_SNAKE_CASE__													= os.path.join(self.tmpdirname       ,   VOCAB_FILES_NAMES['vocab_file']							)
						SCREAMING_SNAKE_CASE__													= os.path.join(self.tmpdirname       ,   VOCAB_FILES_NAMES['emoji_file']							)
						with open(self.vocab_file       ,   'w'       ,   encoding='utf-8'							) as vocab_writer:
											vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]							)							)
						with open(self.emoji_file       ,   'w'							) as emoji_writer:
											emoji_writer.write(json.dumps(UpperCAmelCase_							)							)
	def      A_				(			self							:   str       ,   **UpperCAmelCase_							:   Optional[Any]							):
						kwargs.update(self.special_tokens_map							)
						return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname       ,   **UpperCAmelCase_							)
	def      A_				(			self							:   int       ,   UpperCAmelCase_							:   List[str]							):
						SCREAMING_SNAKE_CASE__													= 'こんにちは、世界。 \nこんばんは、㔺界。😀'
						SCREAMING_SNAKE_CASE__													= 'こんにちは、世界。 \nこんばんは、世界。😀'
						return input_text, output_text
	def      A_				(			self							:   Any       ,   UpperCAmelCase_							:   str							):
						SCREAMING_SNAKE_CASE__						, SCREAMING_SNAKE_CASE__													= self.get_input_output_texts(UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= tokenizer.encode(UpperCAmelCase_       ,   add_special_tokens=UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= tokenizer.decode(UpperCAmelCase_       ,   clean_up_tokenization_spaces=UpperCAmelCase_							)
						return text, ids
	def      A_				(			self							:   str							):
						pass  # TODO add if relevant
	def      A_				(			self							:   Tuple							):
						pass  # TODO add if relevant
	def      A_				(			self							:   int							):
						pass  # TODO add if relevant
	def      A_				(			self							:   Optional[int]							):
						SCREAMING_SNAKE_CASE__													= self.get_tokenizer()

						# Testing tokenization
						SCREAMING_SNAKE_CASE__													= 'こんにちは、世界。　こんばんは、㔺界。'
						SCREAMING_SNAKE_CASE__													= ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
						SCREAMING_SNAKE_CASE__													= tokenizer.tokenize(UpperCAmelCase_							)
						self.assertListEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)

						# Testing conversion to ids without special tokens
						SCREAMING_SNAKE_CASE__													= [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
						SCREAMING_SNAKE_CASE__													= tokenizer.convert_tokens_to_ids(UpperCAmelCase_							)
						self.assertListEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)

						# Testing conversion to ids with special tokens
						SCREAMING_SNAKE_CASE__													= tokens + [tokenizer.unk_token]
						SCREAMING_SNAKE_CASE__													= [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
						SCREAMING_SNAKE_CASE__													= tokenizer.convert_tokens_to_ids(UpperCAmelCase_							)
						self.assertListEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)
	def      A_				(			self							:   Union[str, Any]							):
						SCREAMING_SNAKE_CASE__													= self.get_tokenizer()

						# Testing tokenization
						SCREAMING_SNAKE_CASE__													= 'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
						SCREAMING_SNAKE_CASE__													= 'こんにちは、、、、世界。こんばんは、、、、世界。'
						SCREAMING_SNAKE_CASE__													= tokenizer.encode(UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= tokenizer.decode(UpperCAmelCase_							)
						self.assertEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)
	@slow
	def      A_				(			self							:   str							):
						SCREAMING_SNAKE_CASE__													= self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'							)

						# Testing tokenization
						SCREAMING_SNAKE_CASE__													= 'こんにちは、世界。'
						SCREAMING_SNAKE_CASE__													= 'こんばんは、㔺界。😀'
						SCREAMING_SNAKE_CASE__													= 'こんにちは、世界。こんばんは、世界。😀'
						SCREAMING_SNAKE_CASE__													= tokenizer.encode(prefix_text + input_text							)
						SCREAMING_SNAKE_CASE__													= tokenizer.encode(''       ,   prefix_text=prefix_text + input_text							)
						SCREAMING_SNAKE_CASE__													= tokenizer.encode(UpperCAmelCase_       ,   prefix_text=UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= tokenizer.decode(UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= tokenizer.decode(UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= tokenizer.decode(UpperCAmelCase_							)
						self.assertEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)
						self.assertEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)
						self.assertEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)
	@slow
	def      A_				(			self							:   Union[str, Any]							):
						SCREAMING_SNAKE_CASE__													= self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'							)

						# Testing tokenization
						SCREAMING_SNAKE_CASE__													= 'こんにちは、世界。'
						SCREAMING_SNAKE_CASE__													= 'こんばんは、㔺界。😀'

						SCREAMING_SNAKE_CASE__													= len(tokenizer.encode(UpperCAmelCase_							)							) - 2
						SCREAMING_SNAKE_CASE__													= len(tokenizer.encode(UpperCAmelCase_							)							) - 2

						SCREAMING_SNAKE_CASE__													= [1] + [0] * (len_prefix + len_text + 1)
						SCREAMING_SNAKE_CASE__													= [1] * (len_prefix + len_text + 1) + [0]
						SCREAMING_SNAKE_CASE__													= [1] + [1] * (len_prefix) + [0] * (len_text + 1)

						SCREAMING_SNAKE_CASE__													= tokenizer(prefix_text + input_text							).token_type_ids
						SCREAMING_SNAKE_CASE__													= tokenizer(''       ,   prefix_text=prefix_text + input_text							).token_type_ids
						SCREAMING_SNAKE_CASE__													= tokenizer(UpperCAmelCase_       ,   prefix_text=UpperCAmelCase_							).token_type_ids
						self.assertListEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)
						self.assertListEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)
						self.assertListEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)
	@slow
	def      A_				(			self							:   Tuple							):
						SCREAMING_SNAKE_CASE__													= self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'							)

						SCREAMING_SNAKE_CASE__													= tokenizer.encode('あンいワ'							)
						SCREAMING_SNAKE_CASE__													= tokenizer.encode(''       ,   prefix_text='あンいワ'							)
						SCREAMING_SNAKE_CASE__													= tokenizer.encode('いワ'       ,   prefix_text='あン'							)

						self.assertEqual(tokenizer.decode(UpperCAmelCase_							)       ,   tokenizer.decode(UpperCAmelCase_							)							)
						self.assertEqual(tokenizer.decode(UpperCAmelCase_							)       ,   tokenizer.decode(UpperCAmelCase_							)							)
						self.assertNotEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)
						self.assertNotEqual(UpperCAmelCase_       ,   UpperCAmelCase_							)
						self.assertEqual(x_token_a[1]       ,   x_token_a[-1]							)  # SEG token
						self.assertEqual(x_token_a[1]       ,   x_token_a[3]							)  # SEG token
	@slow
	def      A_				(			self							:   Union[str, Any]							):
						SCREAMING_SNAKE_CASE__													= self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'							)

						SCREAMING_SNAKE_CASE__													= [['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
						SCREAMING_SNAKE_CASE__													= tokenizer(UpperCAmelCase_       ,   padding=UpperCAmelCase_							)
						SCREAMING_SNAKE_CASE__													= tokenizer.batch_encode_plus(UpperCAmelCase_       ,   padding=UpperCAmelCase_							)

						# fmt: off
						SCREAMING_SNAKE_CASE__													= [[35993, 8640, 25948, 35998, 30647, 35675, 35999, 35999], [35993, 10382, 9868, 35998, 30646, 9459, 30646, 35675]]
						SCREAMING_SNAKE_CASE__													= [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
						SCREAMING_SNAKE_CASE__													= [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
						# fmt: on
						self.assertListEqual(x_token.input_ids       ,   UpperCAmelCase_							)
						self.assertListEqual(x_token.token_type_ids       ,   UpperCAmelCase_							)
						self.assertListEqual(x_token.attention_mask       ,   UpperCAmelCase_							)
						self.assertListEqual(x_token_a.input_ids       ,   UpperCAmelCase_							)
						self.assertListEqual(x_token_a.token_type_ids       ,   UpperCAmelCase_							)
						self.assertListEqual(x_token_a.attention_mask       ,   UpperCAmelCase_							)
	def      A_				(			self							:   Tuple							):
						# Intentionally convert some words to accommodate character fluctuations unique to Japanese
						pass





	def      A_				(			self							:   List[str]							):
						# tokenizer has no padding token
						pass





