


import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class        snake_case_   (_UpperCAmelCase	,						unittest.TestCase      ):
					UpperCAmelCase__					:    Any          =			GPTSanJapaneseTokenizer
					UpperCAmelCase__					:    str          =			False
					UpperCAmelCase__					:    int          =			{"""do_clean_text""": False, """add_prefix_space""": False}
					def  lowerCamelCase__(      self   :Any      )					->    str:
												super().setUp()

												# fmt: off
												a__             =     ['こん', 'こんに', 'にちは', 'ばんは', '世界,㔺界', '、', '。', '<BR>', '<SP>', '<TAB>', '<URL>', '<EMAIL>', '<TEL>', '<DATE>', '<PRICE>', '<BLOCK>', '<KIGOU>', '<U2000U2BFF>', '<|emoji1|>', '<unk>', '<|bagoftoken|>', '<|endoftext|>']
												# fmt: on
												a__             =     {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
												a__             =     {'unk_token': '<unk>'}

												a__             =     os.path.join(self.tmpdirname			,VOCAB_FILES_NAMES['vocab_file']      )
												a__             =     os.path.join(self.tmpdirname			,VOCAB_FILES_NAMES['emoji_file']      )
												with open(self.vocab_file			,'w'			,encoding='utf-8'      ) as vocab_writer:
																			vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]      )      )
												with open(self.emoji_file			,'w'      ) as emoji_writer:
																			emoji_writer.write(json.dumps(UpperCAmelCase_      )      )
					def  lowerCamelCase__(      self   :str			,**__snake_case   :Optional[Any]      )					->    Optional[Any]:
												kwargs.update(self.special_tokens_map      )
												return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname			,**UpperCAmelCase_      )
					def  lowerCamelCase__(      self   :int			,__snake_case   :List[str]      )					->    str:
												a__             =     'こんにちは、世界。 \nこんばんは、㔺界。😀'
												a__             =     'こんにちは、世界。 \nこんばんは、世界。😀'
												return input_text, output_text
					def  lowerCamelCase__(      self   :Any			,__snake_case   :str      )					->    int:
												a__      ,					a__             =     self.get_input_output_texts(UpperCAmelCase_      )
												a__             =     tokenizer.encode(UpperCAmelCase_			,add_special_tokens=UpperCAmelCase_      )
												a__             =     tokenizer.decode(UpperCAmelCase_			,clean_up_tokenization_spaces=UpperCAmelCase_      )
												return text, ids
					def  lowerCamelCase__(      self   :str      )					->    Optional[int]:
												pass  # TODO add if relevant
					def  lowerCamelCase__(      self   :Tuple      )					->    Optional[int]:
												pass  # TODO add if relevant
					def  lowerCamelCase__(      self   :int      )					->    List[str]:
												pass  # TODO add if relevant
					def  lowerCamelCase__(      self   :Optional[int]      )					->    Dict:
												a__             =     self.get_tokenizer()

												# Testing tokenization
												a__             =     'こんにちは、世界。　こんばんは、㔺界。'
												a__             =     ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
												a__             =     tokenizer.tokenize(UpperCAmelCase_      )
												self.assertListEqual(UpperCAmelCase_			,UpperCAmelCase_      )

												# Testing conversion to ids without special tokens
												a__             =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
												a__             =     tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )
												self.assertListEqual(UpperCAmelCase_			,UpperCAmelCase_      )

												# Testing conversion to ids with special tokens
												a__             =     tokens + [tokenizer.unk_token]
												a__             =     [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
												a__             =     tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )
												self.assertListEqual(UpperCAmelCase_			,UpperCAmelCase_      )
					def  lowerCamelCase__(      self   :Union[str, Any]      )					->    List[str]:
												a__             =     self.get_tokenizer()

												# Testing tokenization
												a__             =     'こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'
												a__             =     'こんにちは、、、、世界。こんばんは、、、、世界。'
												a__             =     tokenizer.encode(UpperCAmelCase_      )
												a__             =     tokenizer.decode(UpperCAmelCase_      )
												self.assertEqual(UpperCAmelCase_			,UpperCAmelCase_      )
					@slow
					def  lowerCamelCase__(      self   :str      )					->    Tuple:
												a__             =     self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'      )

												# Testing tokenization
												a__             =     'こんにちは、世界。'
												a__             =     'こんばんは、㔺界。😀'
												a__             =     'こんにちは、世界。こんばんは、世界。😀'
												a__             =     tokenizer.encode(prefix_text + input_text      )
												a__             =     tokenizer.encode(''			,prefix_text=prefix_text + input_text      )
												a__             =     tokenizer.encode(UpperCAmelCase_			,prefix_text=UpperCAmelCase_      )
												a__             =     tokenizer.decode(UpperCAmelCase_      )
												a__             =     tokenizer.decode(UpperCAmelCase_      )
												a__             =     tokenizer.decode(UpperCAmelCase_      )
												self.assertEqual(UpperCAmelCase_			,UpperCAmelCase_      )
												self.assertEqual(UpperCAmelCase_			,UpperCAmelCase_      )
												self.assertEqual(UpperCAmelCase_			,UpperCAmelCase_      )
					@slow
					def  lowerCamelCase__(      self   :Union[str, Any]      )					->    Optional[int]:
												a__             =     self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'      )

												# Testing tokenization
												a__             =     'こんにちは、世界。'
												a__             =     'こんばんは、㔺界。😀'

												a__             =     len(tokenizer.encode(UpperCAmelCase_      )      ) - 2
												a__             =     len(tokenizer.encode(UpperCAmelCase_      )      ) - 2

												a__             =     [1] + [0] * (len_prefix + len_text + 1)
												a__             =     [1] * (len_prefix + len_text + 1) + [0]
												a__             =     [1] + [1] * (len_prefix) + [0] * (len_text + 1)

												a__             =     tokenizer(prefix_text + input_text      ).token_type_ids
												a__             =     tokenizer(''			,prefix_text=prefix_text + input_text      ).token_type_ids
												a__             =     tokenizer(UpperCAmelCase_			,prefix_text=UpperCAmelCase_      ).token_type_ids
												self.assertListEqual(UpperCAmelCase_			,UpperCAmelCase_      )
												self.assertListEqual(UpperCAmelCase_			,UpperCAmelCase_      )
												self.assertListEqual(UpperCAmelCase_			,UpperCAmelCase_      )
					@slow
					def  lowerCamelCase__(      self   :Tuple      )					->    Tuple:
												a__             =     self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'      )

												a__             =     tokenizer.encode('あンいワ'      )
												a__             =     tokenizer.encode(''			,prefix_text='あンいワ'      )
												a__             =     tokenizer.encode('いワ'			,prefix_text='あン'      )

												self.assertEqual(tokenizer.decode(UpperCAmelCase_      )			,tokenizer.decode(UpperCAmelCase_      )      )
												self.assertEqual(tokenizer.decode(UpperCAmelCase_      )			,tokenizer.decode(UpperCAmelCase_      )      )
												self.assertNotEqual(UpperCAmelCase_			,UpperCAmelCase_      )
												self.assertNotEqual(UpperCAmelCase_			,UpperCAmelCase_      )
												self.assertEqual(x_token_a[1]			,x_token_a[-1]      )  # SEG token
												self.assertEqual(x_token_a[1]			,x_token_a[3]      )  # SEG token
					@slow
					def  lowerCamelCase__(      self   :Union[str, Any]      )					->    str:
												a__             =     self.tokenizer_class.from_pretrained('Tanrei/GPTSAN-japanese'      )

												a__             =     [['武田信玄', 'は、'], ['織田信長', 'の配下の、']]
												a__             =     tokenizer(UpperCAmelCase_			,padding=UpperCAmelCase_      )
												a__             =     tokenizer.batch_encode_plus(UpperCAmelCase_			,padding=UpperCAmelCase_      )

												# fmt: off
												a__             =     [[3_59_93, 86_40, 2_59_48, 3_59_98, 3_06_47, 3_56_75, 3_59_99, 3_59_99], [3_59_93, 1_03_82, 98_68, 3_59_98, 3_06_46, 94_59, 3_06_46, 3_56_75]]
												a__             =     [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
												a__             =     [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
												# fmt: on
												self.assertListEqual(x_token.input_ids			,UpperCAmelCase_      )
												self.assertListEqual(x_token.token_type_ids			,UpperCAmelCase_      )
												self.assertListEqual(x_token.attention_mask			,UpperCAmelCase_      )
												self.assertListEqual(x_token_a.input_ids			,UpperCAmelCase_      )
												self.assertListEqual(x_token_a.token_type_ids			,UpperCAmelCase_      )
												self.assertListEqual(x_token_a.attention_mask			,UpperCAmelCase_      )
					def  lowerCamelCase__(      self   :Tuple      )					->    Union[str, Any]:
												# Intentionally convert some words to accommodate character fluctuations unique to Japanese
												pass



					def  lowerCamelCase__(      self   :List[str]      )					->    Any:
												# tokenizer has no padding token
												pass






import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case								=  get_tests_dir("""fixtures/test_sentencepiece.model""")




@require_sentencepiece
class 						lowercase__  (						_UpperCAmelCase     ,       unittest.TestCase  ):
	A__    :   List[str]				       =XLMProphetNetTokenizer
	A__    :   List[Any]				       =False
	A__    :   Tuple				       =True
	def      A_				(			self							:   str							):
						super().setUp()

						# We have a SentencePiece fixture for testing
						SCREAMING_SNAKE_CASE__													= XLMProphetNetTokenizer(UpperCAmelCase_       ,   keep_accents=UpperCAmelCase_							)
						tokenizer.save_pretrained(self.tmpdirname							)
	def      A_				(			self							:   Tuple							):
						SCREAMING_SNAKE_CASE__													= '[PAD]'
						SCREAMING_SNAKE_CASE__													= 0

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase_							)       ,   UpperCAmelCase_							)
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase_							)       ,   UpperCAmelCase_							)
	def      A_				(			self							:   Tuple							):
						SCREAMING_SNAKE_CASE__													= list(self.get_tokenizer().get_vocab().keys()							)

						self.assertEqual(vocab_keys[0]       ,   '[PAD]'							)
						self.assertEqual(vocab_keys[1]       ,   '[CLS]'							)
						self.assertEqual(vocab_keys[-1]       ,   'j'							)
						self.assertEqual(len(UpperCAmelCase_							)       ,   1012							)
	def      A_				(			self							:   List[Any]							):
						self.assertEqual(self.get_tokenizer().vocab_size       ,   1012							)
	def      A_				(			self							:   Optional[int]							):
						SCREAMING_SNAKE_CASE__													= XLMProphetNetTokenizer(UpperCAmelCase_       ,   keep_accents=UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= tokenizer.tokenize('This is a test'							)
						self.assertListEqual(UpperCAmelCase_       ,   ['▁This', '▁is', '▁a', '▁t', 'est']							)

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(UpperCAmelCase_							)       ,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,   )

						SCREAMING_SNAKE_CASE__													= tokenizer.tokenize('I was born in 92000, and this is falsé.'							)
						self.assertListEqual(
						    UpperCAmelCase_       ,   [
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '9',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        'é',
						        '.',
						    ]       ,   )
						SCREAMING_SNAKE_CASE__													= tokenizer.convert_tokens_to_ids(UpperCAmelCase_							)
						self.assertListEqual(
						    UpperCAmelCase_       ,   [
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
						    ]       ,   )

						SCREAMING_SNAKE_CASE__													= tokenizer.convert_ids_to_tokens(UpperCAmelCase_							)
						self.assertListEqual(
						    UpperCAmelCase_       ,   [
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '[UNK]',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        '[UNK]',
						        '.',
						    ]       ,   )
	@cached_property
	def      A_				(			self							:   Optional[Any]							):
						return XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased'							)
	@slow
	def      A_				(			self							:   Any							):
						SCREAMING_SNAKE_CASE__													= 'Hello World!'
						SCREAMING_SNAKE_CASE__													= [35389, 6672, 49, 2]
						self.assertListEqual(UpperCAmelCase_       ,   self.big_tokenizer.encode(UpperCAmelCase_							)							)





	@slow
	def      A_				(			self							:   Tuple							):
						# fmt: off
						SCREAMING_SNAKE_CASE__													= {'input_ids': [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
						# fmt: on

						self.tokenizer_integration_test_util(
						    expected_encoding=UpperCAmelCase_       ,   model_name='microsoft/xprophetnet-large-wiki100-cased'       ,   revision='1acad1643ddd54a44df6a1b797ada8373685d90e'       ,   )





