





"""simple docstring"""



import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin






@custom_tokenizers
class 		__snake_case					(		SCREAMING_SNAKE_CASE__							,							unittest.TestCase     ):





 """simple docstring"""


 _lowerCamelCase        =     BertJapaneseTokenizer
 _lowerCamelCase        =     False
 _lowerCamelCase        =     True

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     super().setUp()

     __A    :						int						 =     [
         '''[UNK]''',
         '''[CLS]''',
         '''[SEP]''',
         '''こんにちは''',
         '''こん''',
         '''にちは''',
         '''ばんは''',
         '''##こん''',
         '''##にちは''',
         '''##ばんは''',
         '''世界''',
         '''##世界''',
         '''、''',
         '''##、''',
         '''。''',
         '''##。''',
     ]

     __A    :						List[str]						 =     os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES['''vocab_file''']  )
     with open(self.vocab_file					,						'''w'''					,						encoding='''utf-8'''  ) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )

 def 					UpperCamelCase__(    self					,						__lowerCamelCase  ):



     '''simple docstring'''




     __A    :						Optional[int]						 =     '''こんにちは、世界。 \nこんばんは、世界。'''
     __A    :						Optional[int]						 =     '''こんにちは 、 世界 。 こんばんは 、 世界 。'''
     return input_text, output_text

 def 					UpperCamelCase__(    self					,						__lowerCamelCase  ):



     '''simple docstring'''




     __A ,       __A    :						Tuple						 =     self.get_input_output_texts(__lowerCamelCase  )
     __A    :						Optional[Any]						 =     tokenizer.encode(__lowerCamelCase					,						add_special_tokens=__lowerCamelCase  )
     __A    :						Tuple						 =     tokenizer.decode(__lowerCamelCase					,						clean_up_tokenization_spaces=__lowerCamelCase  )
     return text, ids

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     pass  # TODO add if relevant

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     pass  # TODO add if relevant

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     pass  # TODO add if relevant

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						str						 =     self.tokenizer_class(self.vocab_file  )

     __A    :						List[Any]						 =     tokenizer.tokenize('''こんにちは、世界。\nこんばんは、世界。'''  )
     self.assertListEqual(__lowerCamelCase					,						['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']  )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase  )					,						[3, 12, 10, 14, 4, 9, 12, 10, 14]  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Tuple						 =     self.tokenizer_class(self.vocab_file					,						word_tokenizer_type='''mecab'''  )
     self.assertIsNotNone(__lowerCamelCase  )

     __A    :						Optional[int]						 =     '''こんにちは、世界。\nこんばんは、世界。'''
     __A    :						Tuple						 =     tokenizer.tokenize(__lowerCamelCase  )
     self.assertListEqual(__lowerCamelCase					,						['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']  )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase  )					,						[3, 12, 10, 14, 4, 9, 12, 10, 14]  )

     __A    :						Any						 =     os.path.join(self.tmpdirname					,						'''tokenizer.bin'''  )
     with open(__lowerCamelCase					,						'''wb'''  ) as handle:
         pickle.dump(__lowerCamelCase					,						__lowerCamelCase  )

     with open(__lowerCamelCase					,						'''rb'''  ) as handle:
         __A    :						Union[str, Any]						 =     pickle.load(__lowerCamelCase  )

     __A    :						str						 =     tokenizer_new.tokenize(__lowerCamelCase  )

     self.assertListEqual(__lowerCamelCase					,						__lowerCamelCase  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Any						 =     MecabTokenizer(mecab_dic='''ipadic'''  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''  )					,						['''アップルストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']					,						)

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     try:
         __A    :						int						 =     MecabTokenizer(mecab_dic='''unidic_lite'''  )
     except ModuleNotFoundError:
         return

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''  )					,						['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']					,						)

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     try:
         __A    :						List[Any]						 =     MecabTokenizer(mecab_dic='''unidic'''  )
     except ModuleNotFoundError:
         return

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''  )					,						['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']					,						)

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Dict						 =     MecabTokenizer(do_lower_case=__lowerCamelCase					,						mecab_dic='''ipadic'''  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''  )					,						['''アップルストア''', '''で''', '''iphone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']					,						)

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     try:
         __A    :						str						 =     MecabTokenizer(
             do_lower_case=__lowerCamelCase					,						normalize_text=__lowerCamelCase					,						mecab_option='''-d /usr/local/lib/mecab/dic/jumandic'''  )
     except RuntimeError:
         # if dict doesn't exist in the system, previous code raises this error.
         return

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''  )					,						['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']					,						)

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						List[Any]						 =     MecabTokenizer(normalize_text=__lowerCamelCase					,						mecab_dic='''ipadic'''  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''  )					,						['''ｱｯﾌﾟﾙストア''', '''で''', '''iPhone''', '''８''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''　''', '''。''']					,						)

 @require_sudachi
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Optional[int]						 =     self.tokenizer_class(self.vocab_file					,						word_tokenizer_type='''sudachi'''  )
     self.assertIsNotNone(__lowerCamelCase  )

     __A    :						Any						 =     '''こんにちは、世界。\nこんばんは、世界。'''
     __A    :						Any						 =     tokenizer.tokenize(__lowerCamelCase  )
     self.assertListEqual(__lowerCamelCase					,						['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']  )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase  )					,						[3, 12, 10, 14, 4, 9, 12, 10, 14]  )

     __A    :						str						 =     os.path.join(self.tmpdirname					,						'''tokenizer.bin'''  )
     with open(__lowerCamelCase					,						'''wb'''  ) as handle:
         pickle.dump(__lowerCamelCase					,						__lowerCamelCase  )

     with open(__lowerCamelCase					,						'''rb'''  ) as handle:
         __A    :						Optional[Any]						 =     pickle.load(__lowerCamelCase  )

     __A    :						int						 =     tokenizer_new.tokenize(__lowerCamelCase  )

     self.assertListEqual(__lowerCamelCase					,						__lowerCamelCase  )

 @require_sudachi
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Optional[int]						 =     SudachiTokenizer(sudachi_dict_type='''core'''  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''  )					,						[''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']					,						)

 @require_sudachi
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Dict						 =     SudachiTokenizer(sudachi_dict_type='''core'''					,						sudachi_split_mode='''A'''  )

     self.assertListEqual(tokenizer.tokenize('''外国人参政権'''  )					,						['''外国''', '''人''', '''参政''', '''権''']  )

 @require_sudachi
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Any						 =     SudachiTokenizer(sudachi_dict_type='''core'''					,						sudachi_split_mode='''B'''  )

     self.assertListEqual(tokenizer.tokenize('''外国人参政権'''  )					,						['''外国人''', '''参政権''']  )

 @require_sudachi
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Any						 =     SudachiTokenizer(sudachi_dict_type='''core'''					,						sudachi_split_mode='''C'''  )

     self.assertListEqual(tokenizer.tokenize('''外国人参政権'''  )					,						['''外国人参政権''']  )

 @require_sudachi
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						List[str]						 =     SudachiTokenizer(do_lower_case=__lowerCamelCase					,						sudachi_dict_type='''core'''  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''  )					,						[''' ''', '''\t''', '''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', ''' ''', '''。''', ''' ''', ''' ''']					,						)

 @require_sudachi
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						List[Any]						 =     SudachiTokenizer(normalize_text=__lowerCamelCase					,						sudachi_dict_type='''core'''  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''  )					,						[''' ''', '''\t''', '''ｱｯﾌﾟﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', ''' ''', '''が''', ''' ''', ''' ''', '''\n ''', '''発売''', '''さ''', '''れ''', '''た''', '''\u3000''', '''。''', ''' ''', ''' ''']					,						)

 @require_sudachi
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						str						 =     SudachiTokenizer(trim_whitespace=__lowerCamelCase					,						sudachi_dict_type='''core'''  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''  )					,						['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れ''', '''た''', '''。''']					,						)

 @require_jumanpp
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						str						 =     self.tokenizer_class(self.vocab_file					,						word_tokenizer_type='''jumanpp'''  )
     self.assertIsNotNone(__lowerCamelCase  )

     __A    :						Union[str, Any]						 =     '''こんにちは、世界。\nこんばんは、世界。'''
     __A    :						Dict						 =     tokenizer.tokenize(__lowerCamelCase  )
     self.assertListEqual(__lowerCamelCase					,						['''こんにちは''', '''、''', '''世界''', '''。''', '''こん''', '''##ばんは''', '''、''', '''世界''', '''。''']  )
     self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase  )					,						[3, 12, 10, 14, 4, 9, 12, 10, 14]  )

     __A    :						Tuple						 =     os.path.join(self.tmpdirname					,						'''tokenizer.bin'''  )
     with open(__lowerCamelCase					,						'''wb'''  ) as handle:
         pickle.dump(__lowerCamelCase					,						__lowerCamelCase  )

     with open(__lowerCamelCase					,						'''rb'''  ) as handle:
         __A    :						List[str]						 =     pickle.load(__lowerCamelCase  )

     __A    :						List[Any]						 =     tokenizer_new.tokenize(__lowerCamelCase  )

     self.assertListEqual(__lowerCamelCase					,						__lowerCamelCase  )

 @require_jumanpp
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Union[str, Any]						 =     JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''  )					,						['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']					,						)

 @require_jumanpp
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Any						 =     JumanppTokenizer(do_lower_case=__lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''  )					,						['''アップル''', '''ストア''', '''で''', '''iphone''', '''8''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']					,						)

 @require_jumanpp
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Union[str, Any]						 =     JumanppTokenizer(normalize_text=__lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''  )					,						['''ｱ''', '''ｯ''', '''ﾌ''', '''ﾟ''', '''ﾙ''', '''ストア''', '''で''', '''iPhone''', '''８''', '''\u3000''', '''が''', '''\u3000''', '''\u3000''', '''\u3000''', '''発売''', '''さ''', '''れた''', '''\u3000''', '''。''']					,						)

 @require_jumanpp
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						int						 =     JumanppTokenizer(trim_whitespace=__lowerCamelCase  )

     self.assertListEqual(
         tokenizer.tokenize(''' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '''  )					,						['''アップル''', '''ストア''', '''で''', '''iPhone''', '''8''', '''が''', '''発売''', '''さ''', '''れた''', '''。''']					,						)

 @require_jumanpp
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Optional[Any]						 =     JumanppTokenizer()

     self.assertListEqual(
         tokenizer.tokenize('''ありがとうございますm(_ _)ｍ見つけるのが大変です。'''  )					,						['''ありがとう''', '''ございます''', '''m(_ _)m''', '''見つける''', '''の''', '''が''', '''大変です''', '''。''']					,						)

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						int						 =     ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

     __A    :						Optional[int]						 =     {}
     for i, token in enumerate(__lowerCamelCase  ):
         __A    :						Optional[int]						 =     i
     __A    :						List[str]						 =     WordpieceTokenizer(vocab=__lowerCamelCase					,						unk_token='''[UNK]'''  )

     self.assertListEqual(tokenizer.tokenize(''''''  )					,						[]  )

     self.assertListEqual(tokenizer.tokenize('''こんにちは'''  )					,						['''こんにちは''']  )

     self.assertListEqual(tokenizer.tokenize('''こんばんは'''  )					,						['''こん''', '''##ばんは''']  )

     self.assertListEqual(tokenizer.tokenize('''こんばんは こんばんにちは こんにちは'''  )					,						['''こん''', '''##ばんは''', '''[UNK]''', '''こんにちは''']  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Union[str, Any]						 =     BertJapaneseTokenizer.from_pretrained('''nlp-waseda/roberta-base-japanese-with-auto-jumanpp'''  )
     __A    :						List[str]						 =     tokenizer.subword_tokenizer

     __A    :						List[str]						 =     subword_tokenizer.tokenize('''国境 の 長い トンネル を 抜ける と 雪国 であった 。'''  )
     self.assertListEqual(__lowerCamelCase					,						['''▁国境''', '''▁の''', '''▁長い''', '''▁トンネル''', '''▁を''', '''▁抜ける''', '''▁と''', '''▁雪''', '''国''', '''▁であった''', '''▁。''']  )

     __A    :						Tuple						 =     subword_tokenizer.tokenize('''こんばんは こんばん にち は こんにちは'''  )
     self.assertListEqual(__lowerCamelCase					,						['''▁こん''', '''ばん''', '''は''', '''▁こん''', '''ばん''', '''▁に''', '''ち''', '''▁は''', '''▁こんにちは''']  )







 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Optional[int]						 =     self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese'''  )

     __A    :						Union[str, Any]						 =     tokenizer.encode('''ありがとう。'''					,						add_special_tokens=__lowerCamelCase  )
     __A    :						Tuple						 =     tokenizer.encode('''どういたしまして。'''					,						add_special_tokens=__lowerCamelCase  )

     __A    :						Any						 =     tokenizer.build_inputs_with_special_tokens(__lowerCamelCase  )
     __A    :						List[Any]						 =     tokenizer.build_inputs_with_special_tokens(__lowerCamelCase					,						__lowerCamelCase  )

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class 		__snake_case					(		SCREAMING_SNAKE_CASE__							,							unittest.TestCase     ):





 """simple docstring"""


 _lowerCamelCase        =     BertJapaneseTokenizer
 _lowerCamelCase        =     False

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     super().setUp()

     __A    :						Optional[int]						 =     ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

     __A    :						Optional[int]						 =     os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES['''vocab_file''']  )
     with open(self.vocab_file					,						'''w'''					,						encoding='''utf-8'''  ) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )

 def 					UpperCamelCase__(    self					,						**__lowerCamelCase  ):



     '''simple docstring'''




     return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,						subword_tokenizer_type='''character'''					,						**__lowerCamelCase  )

 def 					UpperCamelCase__(    self					,						__lowerCamelCase  ):



     '''simple docstring'''




     __A    :						List[Any]						 =     '''こんにちは、世界。 \nこんばんは、世界。'''
     __A    :						Optional[Any]						 =     '''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
     return input_text, output_text

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     pass  # TODO add if relevant

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     pass  # TODO add if relevant

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     pass  # TODO add if relevant

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Union[str, Any]						 =     self.tokenizer_class(self.vocab_file					,						subword_tokenizer_type='''character'''  )

     __A    :						Optional[int]						 =     tokenizer.tokenize('''こんにちは、世界。 \nこんばんは、世界。'''  )
     self.assertListEqual(
         __lowerCamelCase					,						['''こ''', '''ん''', '''に''', '''ち''', '''は''', '''、''', '''世''', '''界''', '''。''', '''こ''', '''ん''', '''ば''', '''ん''', '''は''', '''、''', '''世''', '''界''', '''。''']  )
     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(__lowerCamelCase  )					,						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						str						 =     ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

     __A    :						str						 =     {}
     for i, token in enumerate(__lowerCamelCase  ):
         __A    :						Union[str, Any]						 =     i
     __A    :						Optional[int]						 =     CharacterTokenizer(vocab=__lowerCamelCase					,						unk_token='''[UNK]'''  )

     self.assertListEqual(tokenizer.tokenize(''''''  )					,						[]  )

     self.assertListEqual(tokenizer.tokenize('''こんにちは'''  )					,						['''こ''', '''ん''', '''に''', '''ち''', '''は''']  )

     self.assertListEqual(tokenizer.tokenize('''こんにちほ'''  )					,						['''こ''', '''ん''', '''に''', '''ち''', '''[UNK]''']  )







 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Union[str, Any]						 =     self.tokenizer_class.from_pretrained('''cl-tohoku/bert-base-japanese-char'''  )

     __A    :						List[Any]						 =     tokenizer.encode('''ありがとう。'''					,						add_special_tokens=__lowerCamelCase  )
     __A    :						List[Any]						 =     tokenizer.encode('''どういたしまして。'''					,						add_special_tokens=__lowerCamelCase  )

     __A    :						Optional[Any]						 =     tokenizer.build_inputs_with_special_tokens(__lowerCamelCase  )
     __A    :						Union[str, Any]						 =     tokenizer.build_inputs_with_special_tokens(__lowerCamelCase					,						__lowerCamelCase  )

     # 2 is for "[CLS]", 3 is for "[SEP]"
     assert encoded_sentence == [2] + text + [3]
     assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class 		__snake_case					(		unittest.TestCase     ):





 """simple docstring"""


 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Optional[Any]						 =     '''cl-tohoku/bert-base-japanese'''
     __A    :						List[Any]						 =     AutoTokenizer.from_pretrained(__lowerCamelCase  )
     self.assertIsInstance(__lowerCamelCase					,						__lowerCamelCase  )






class 		__snake_case					(		unittest.TestCase     ):





 """simple docstring"""


 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						List[str]						 =     '''cl-tohoku/bert-base-japanese'''
     with self.assertLogs('''transformers'''					,						level='''WARNING'''  ) as cm:
         BertTokenizer.from_pretrained(__lowerCamelCase  )
         self.assertTrue(
             cm.records[0].message.startswith(
                 '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
                 ''' is called from.'''  )  )
     __A    :						List[str]						 =     '''bert-base-cased'''
     with self.assertLogs('''transformers'''					,						level='''WARNING'''  ) as cm:
         BertJapaneseTokenizer.from_pretrained(__lowerCamelCase  )
         self.assertTrue(
             cm.records[0].message.startswith(
                 '''The tokenizer class you load from this checkpoint is not the same type as the class this function'''
                 ''' is called from.'''  )  )











"""simple docstring"""



import unittest

import numpy as np
import requests

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
       import torch

       from transformers.pytorch_utils import is_torch_greater_or_equal_than_1_11
else:
       a_ =			False

if is_vision_available():
       from PIL import Image

       from transformers import PixaStructImageProcessor






class 		__snake_case					(		unittest.TestCase     ):





 """simple docstring"""


 def __init__(    self					,						__lowerCamelCase					,						__lowerCamelCase=7					,						__lowerCamelCase=3					,						__lowerCamelCase=18					,						__lowerCamelCase=30					,						__lowerCamelCase=400					,						__lowerCamelCase=None					,						__lowerCamelCase=True					,						__lowerCamelCase=True					,						__lowerCamelCase=None					,						):



     '''simple docstring'''




     __A    :						str						 =     size if size is not None else {'''height''': 20, '''width''': 20}
     __A    :						Dict						 =     parent
     __A    :						Union[str, Any]						 =     batch_size
     __A    :						List[Any]						 =     num_channels
     __A    :						Union[str, Any]						 =     image_size
     __A    :						Any						 =     min_resolution
     __A    :						str						 =     max_resolution
     __A    :						Any						 =     size
     __A    :						Dict						 =     do_normalize
     __A    :						str						 =     do_convert_rgb
     __A    :						List[Any]						 =     [512, 1024, 2048, 4096]
     __A    :						Dict						 =     patch_size if patch_size is not None else {'''height''': 16, '''width''': 16}

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     return {"do_normalize": self.do_normalize, "do_convert_rgb": self.do_convert_rgb}







 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						List[str]						 =     '''https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/australia.jpg'''
     __A    :						Optional[Any]						 =     Image.open(requests.get(__lowerCamelCase					,						stream=__lowerCamelCase  ).raw  ).convert('''RGB'''  )
     return raw_image






@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11							,							reason="""`Pix2StructImageProcessor` requires `torch>=1.11.0`."""							,							)
@require_torch
@require_vision
class 		__snake_case					(		SCREAMING_SNAKE_CASE__							,							unittest.TestCase     ):





 """simple docstring"""


 _lowerCamelCase        =     PixaStructImageProcessor if is_vision_available() else None

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						str						 =     PixaStructImageProcessingTester(self  )

 @property
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     return self.image_processor_tester.prepare_image_processor_dict()

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Any						 =     self.image_processing_class(**self.image_processor_dict  )
     self.assertTrue(hasattr(__lowerCamelCase					,						'''do_normalize'''  )  )
     self.assertTrue(hasattr(__lowerCamelCase					,						'''do_convert_rgb'''  )  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Tuple						 =     self.image_processor_tester.prepare_dummy_image()

     __A    :						Optional[int]						 =     self.image_processing_class(**self.image_processor_dict  )
     __A    :						Dict						 =     2048

     __A    :						Any						 =     image_processor(__lowerCamelCase					,						return_tensors='''pt'''					,						max_patches=__lowerCamelCase  )
     self.assertTrue(torch.allclose(inputs.flattened_patches.mean()					,						torch.tensor(0.0_6_0_6  )					,						atol=1e-3					,						rtol=1e-3  )  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						List[Any]						 =     self.image_processing_class(**self.image_processor_dict  )
     # create random PIL images
     __A    :						str						 =     prepare_image_inputs(self.image_processor_tester					,						equal_resolution=__lowerCamelCase  )
     for image in image_inputs:
         self.assertIsInstance(__lowerCamelCase					,						Image.Image  )

     # Test not batched input
     __A    :						List[str]						 =     (
         (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
         * self.image_processor_tester.num_channels
     ) + 2

     for max_patch in self.image_processor_tester.max_patches:
         # Test not batched input
         __A    :						Dict						 =     image_processor(
             image_inputs[0]					,						return_tensors='''pt'''					,						max_patches=__lowerCamelCase  ).flattened_patches
         self.assertEqual(
             encoded_images.shape					,						(1, max_patch, expected_hidden_dim)					,						)

         # Test batched
         __A    :						Tuple						 =     image_processor(
             __lowerCamelCase					,						return_tensors='''pt'''					,						max_patches=__lowerCamelCase  ).flattened_patches
         self.assertEqual(
             encoded_images.shape					,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)					,						)

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						List[str]						 =     self.image_processing_class(**self.image_processor_dict  )
     # create random PIL images
     __A    :						Union[str, Any]						 =     prepare_image_inputs(self.image_processor_tester					,						equal_resolution=__lowerCamelCase  )
     for image in image_inputs:
         self.assertIsInstance(__lowerCamelCase					,						Image.Image  )

     # Test not batched input
     __A    :						Tuple						 =     (
         (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
         * self.image_processor_tester.num_channels
     ) + 2

     __A    :						str						 =     True

     for max_patch in self.image_processor_tester.max_patches:
         # Test not batched input
         with self.assertRaises(__lowerCamelCase  ):
             __A    :						List[Any]						 =     image_processor(
                 image_inputs[0]					,						return_tensors='''pt'''					,						max_patches=__lowerCamelCase  ).flattened_patches

         __A    :						Any						 =     '''Hello'''

         __A    :						Tuple						 =     image_processor(
             image_inputs[0]					,						return_tensors='''pt'''					,						max_patches=__lowerCamelCase					,						header_text=__lowerCamelCase  ).flattened_patches
         self.assertEqual(
             encoded_images.shape					,						(1, max_patch, expected_hidden_dim)					,						)

         # Test batched
         __A    :						Union[str, Any]						 =     image_processor(
             __lowerCamelCase					,						return_tensors='''pt'''					,						max_patches=__lowerCamelCase					,						header_text=__lowerCamelCase  ).flattened_patches
         self.assertEqual(
             encoded_images.shape					,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)					,						)

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						int						 =     self.image_processing_class(**self.image_processor_dict  )
     # create random numpy tensors
     __A    :						Dict						 =     prepare_image_inputs(self.image_processor_tester					,						equal_resolution=__lowerCamelCase					,						numpify=__lowerCamelCase  )
     for image in image_inputs:
         self.assertIsInstance(__lowerCamelCase					,						np.ndarray  )

     __A    :						List[Any]						 =     (
         (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
         * self.image_processor_tester.num_channels
     ) + 2

     for max_patch in self.image_processor_tester.max_patches:
         # Test not batched input
         __A    :						List[str]						 =     image_processor(
             image_inputs[0]					,						return_tensors='''pt'''					,						max_patches=__lowerCamelCase  ).flattened_patches
         self.assertEqual(
             encoded_images.shape					,						(1, max_patch, expected_hidden_dim)					,						)

         # Test batched
         __A    :						Union[str, Any]						 =     image_processor(
             __lowerCamelCase					,						return_tensors='''pt'''					,						max_patches=__lowerCamelCase  ).flattened_patches
         self.assertEqual(
             encoded_images.shape					,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)					,						)







 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						int						 =     self.image_processing_class(**self.image_processor_dict  )
     # create random PyTorch tensors
     __A    :						Optional[Any]						 =     prepare_image_inputs(self.image_processor_tester					,						equal_resolution=__lowerCamelCase					,						torchify=__lowerCamelCase  )
     for image in image_inputs:
         self.assertIsInstance(__lowerCamelCase					,						torch.Tensor  )

     # Test not batched input
     __A    :						int						 =     (
         (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
         * self.image_processor_tester.num_channels
     ) + 2

     for max_patch in self.image_processor_tester.max_patches:
         # Test not batched input
         __A    :						List[Any]						 =     image_processor(
             image_inputs[0]					,						return_tensors='''pt'''					,						max_patches=__lowerCamelCase  ).flattened_patches
         self.assertEqual(
             encoded_images.shape					,						(1, max_patch, expected_hidden_dim)					,						)

         # Test batched
         __A    :						Dict						 =     image_processor(
             __lowerCamelCase					,						return_tensors='''pt'''					,						max_patches=__lowerCamelCase  ).flattened_patches
         self.assertEqual(
             encoded_images.shape					,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)					,						)






@unittest.skipIf(
    not is_torch_greater_or_equal_than_1_11							,							reason="""`Pix2StructImageProcessor` requires `torch>=1.11.0`."""							,							)
@require_torch
@require_vision
class 		__snake_case					(		SCREAMING_SNAKE_CASE__							,							unittest.TestCase     ):





 """simple docstring"""


 _lowerCamelCase        =     PixaStructImageProcessor if is_vision_available() else None

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						int						 =     PixaStructImageProcessingTester(self					,						num_channels=4  )
     __A    :						Any						 =     3

 @property
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     return self.image_processor_tester.prepare_image_processor_dict()

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Union[str, Any]						 =     self.image_processing_class(**self.image_processor_dict  )
     self.assertTrue(hasattr(__lowerCamelCase					,						'''do_normalize'''  )  )
     self.assertTrue(hasattr(__lowerCamelCase					,						'''do_convert_rgb'''  )  )







 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Optional[int]						 =     self.image_processing_class(**self.image_processor_dict  )
     # create random PIL images
     __A    :						Dict						 =     prepare_image_inputs(self.image_processor_tester					,						equal_resolution=__lowerCamelCase  )
     for image in image_inputs:
         self.assertIsInstance(__lowerCamelCase					,						Image.Image  )

     # Test not batched input
     __A    :						List[Any]						 =     (
         (self.image_processor_tester.patch_size['''height'''] * self.image_processor_tester.patch_size['''width'''])
         * (self.image_processor_tester.num_channels - 1)
     ) + 2

     for max_patch in self.image_processor_tester.max_patches:
         # Test not batched input
         __A    :						Any						 =     image_processor(
             image_inputs[0]					,						return_tensors='''pt'''					,						max_patches=__lowerCamelCase  ).flattened_patches
         self.assertEqual(
             encoded_images.shape					,						(1, max_patch, expected_hidden_dim)					,						)

         # Test batched
         __A    :						int						 =     image_processor(
             __lowerCamelCase					,						return_tensors='''pt'''					,						max_patches=__lowerCamelCase  ).flattened_patches
         self.assertEqual(
             encoded_images.shape					,						(self.image_processor_tester.batch_size, max_patch, expected_hidden_dim)					,						)




