
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


__lowerCAmelCase				      =					{
    'configuration_deta': ['DETA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'DetaConfig'],
}

try:
							if not is_vision_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowerCAmelCase				      =					['DetaImageProcessor']

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowerCAmelCase				      =					[
							    'DETA_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'DetaForObjectDetection',
							    'DetaModel',
							    'DetaPreTrainedModel',
							]


if TYPE_CHECKING:
							from .configuration_deta import DETA_PRETRAINED_CONFIG_ARCHIVE_MAP, DetaConfig

							try:
														if not is_vision_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .image_processing_deta import DetaImageProcessor

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_deta import (
														    DETA_PRETRAINED_MODEL_ARCHIVE_LIST,
														    DetaForObjectDetection,
														    DetaModel,
														    DetaPreTrainedModel,
														)

else:
							import sys

							__lowerCAmelCase				      =					_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 						lowerCamelCase			(					__lowerCamelCase   ,		unittest.TestCase				):
	UpperCamelCase_     :				List[str]								=  GPTNeoXJapaneseTokenizer
	UpperCamelCase_     :				List[Any]								=  False
	UpperCamelCase_     :				int								=  {'do_clean_text': False, 'add_prefix_space': False}







	def    snake_case__    (    self			:Optional[int] )		->			Union[str, Any]:







				"""simple docstring"""






				super().setUp()

				SCREAMING_SNAKE_CASE    	=   [
				    '''こん''',
				    '''こんに''',
				    '''にちは''',
				    '''ばんは''',
				    '''世界,㔺界''',
				    '''、''',
				    '''。''',
				    '''<BR>''',
				    '''<SP>''',
				    '''<TAB>''',
				    '''<URL>''',
				    '''<EMAIL>''',
				    '''<TEL>''',
				    '''<DATE>''',
				    '''<PRICE>''',
				    '''<BLOCK>''',
				    '''<KIGOU>''',
				    '''<U2000U2BFF>''',
				    '''<|emoji1|>''',
				    '''<unk>''',
				    '''<|startoftext|>''',
				    '''<|endoftext|>''',
				]
				SCREAMING_SNAKE_CASE    	=   {'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
				SCREAMING_SNAKE_CASE    	=   {'''unk_token''': '''<unk>'''}

				SCREAMING_SNAKE_CASE    	=   os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES['''vocab_file'''] )
				SCREAMING_SNAKE_CASE    	=   os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES['''emoji_file'''] )
				with open(self.vocab_file      ,    '''w'''      ,    encoding='''utf-8''' ) as vocab_writer:
							vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )
				with open(self.emoji_file      ,    '''w''' ) as emoji_writer:
							emoji_writer.write(json.dumps(lowercase ) )







	def    snake_case__    (    self			:Dict      ,    **lowercase			:Optional[int] )		->			int:







				"""simple docstring"""






				kwargs.update(self.special_tokens_map )
				return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname      ,    **lowercase )







	def    snake_case__    (    self			:Dict      ,    lowercase			:Union[str, Any] )		->			List[str]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE    	=   '''こんにちは、世界。 \nこんばんは、㔺界。😀'''
				SCREAMING_SNAKE_CASE    	=   '''こんにちは、世界。 \nこんばんは、世界。😀'''
				return input_text, output_text







	def    snake_case__    (    self			:Optional[int]      ,    lowercase			:Union[str, Any] )		->			List[str]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE	,       SCREAMING_SNAKE_CASE    	=   self.get_input_output_texts(lowercase )
				SCREAMING_SNAKE_CASE    	=   tokenizer.encode(lowercase      ,    add_special_tokens=lowercase )
				SCREAMING_SNAKE_CASE    	=   tokenizer.decode(lowercase      ,    clean_up_tokenization_spaces=lowercase )
				return text, ids







	def    snake_case__    (    self			:Dict )		->			int:







				"""simple docstring"""






				pass  # TODO add if relevant







	def    snake_case__    (    self			:List[str] )		->			List[str]:







				"""simple docstring"""






				pass  # TODO add if relevant







	def    snake_case__    (    self			:Optional[Any] )		->			Union[str, Any]:







				"""simple docstring"""






				pass  # TODO add if relevant







	def    snake_case__    (    self			:Optional[Any] )		->			Dict:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE    	=   self.get_tokenizer()

				# Testing tokenization
				SCREAMING_SNAKE_CASE    	=   '''こんにちは、世界。　こんばんは、㔺界。'''
				SCREAMING_SNAKE_CASE    	=   ['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
				SCREAMING_SNAKE_CASE    	=   tokenizer.tokenize(lowercase )
				self.assertListEqual(lowercase      ,    lowercase )

				# Testing conversion to ids without special tokens
				SCREAMING_SNAKE_CASE    	=   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
				SCREAMING_SNAKE_CASE    	=   tokenizer.convert_tokens_to_ids(lowercase )
				self.assertListEqual(lowercase      ,    lowercase )

				# Testing conversion to ids with special tokens
				SCREAMING_SNAKE_CASE    	=   tokens + [tokenizer.unk_token]
				SCREAMING_SNAKE_CASE    	=   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
				SCREAMING_SNAKE_CASE    	=   tokenizer.convert_tokens_to_ids(lowercase )
				self.assertListEqual(lowercase      ,    lowercase )







	@slow
	def    snake_case__    (    self			:str )		->			Union[str, Any]:







				"""simple docstring"""






				SCREAMING_SNAKE_CASE    	=   self.tokenizer_class.from_pretrained('''abeja/gpt-neox-japanese-2.7b''' )

				SCREAMING_SNAKE_CASE    	=   tokenizer.encode('''ありがとう。'''      ,    add_special_tokens=lowercase )
				SCREAMING_SNAKE_CASE    	=   tokenizer.encode('''どういたしまして。'''      ,    add_special_tokens=lowercase )

				SCREAMING_SNAKE_CASE    	=   tokenizer.build_inputs_with_special_tokens(lowercase )
				SCREAMING_SNAKE_CASE    	=   tokenizer.build_inputs_with_special_tokens(lowercase      ,    lowercase )

				assert encoded_sentence == ids_a
				assert encoded_pair == ids_a + ids_a







	def    snake_case__    (    self			:List[Any] )		->			Dict:







				"""simple docstring"""






				pass







	def    snake_case__    (    self			:Any )		->			str:







				"""simple docstring"""






				pass