

import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
						from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCAmelCase					:			Union[str, Any]  				=			get_tests_dir("fixtures/test_sentencepiece.model")

_lowerCAmelCase					:			Optional[int]  				=			{"target_lang": "fi", "source_lang": "en"}
_lowerCAmelCase					:			List[Any]  				=			">>zh<<"
_lowerCAmelCase					:			Optional[Any]  				=			"Helsinki-NLP/"

if is_torch_available():
						_lowerCAmelCase					:			Any  				=			"pt"
elif is_tf_available():
						_lowerCAmelCase					:			Any  				=			"tf"
else:
						_lowerCAmelCase					:			str  				=			"jax"






@require_sentencepiece
class 	__magic_name__    (			lowerCAmelCase_  ,    unittest.TestCase      ):
			SCREAMING_SNAKE_CASE							   =						MarianTokenizer
			SCREAMING_SNAKE_CASE							   =						False
			SCREAMING_SNAKE_CASE							   =						True





			def 		__magic_name__    (		self					)   -> Dict:




						'''simple docstring'''
						super().setUp()
						__a        =['</s>', '<unk>', '▁This', '▁is', '▁a', '▁t', 'est', '\u0120', '<pad>']
						__a        =dict(zip(__snake_case					,  range(len(__snake_case					)					)					)					)
						__a        =Path(self.tmpdirname					)
						save_json(__snake_case					,  save_dir / VOCAB_FILES_NAMES['vocab']					)
						save_json(__snake_case					,  save_dir / VOCAB_FILES_NAMES['tokenizer_config_file']					)
						if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
									copyfile(__snake_case					,  save_dir / VOCAB_FILES_NAMES['source_spm']					)
									copyfile(__snake_case					,  save_dir / VOCAB_FILES_NAMES['target_spm']					)

						__a        =MarianTokenizer.from_pretrained(self.tmpdirname					)
						tokenizer.save_pretrained(self.tmpdirname					)





			def 		__magic_name__    (		self					,  **__snake_case					)   -> MarianTokenizer:




						'''simple docstring'''
						return MarianTokenizer.from_pretrained(self.tmpdirname					,  **__snake_case					)





			def 		__magic_name__    (		self					,  __snake_case					)   -> List[Any]:




						'''simple docstring'''
						return (
						    "This is a test",
						    "This is a test",
						)





			def 		__magic_name__    (		self					)   -> Any:




						'''simple docstring'''
						__a        ='</s>'
						__a        =0

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case					)					,  __snake_case					)
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case					)					,  __snake_case					)





			def 		__magic_name__    (		self					)   -> int:




						'''simple docstring'''
						__a        =list(self.get_tokenizer().get_vocab().keys()					)

						self.assertEqual(vocab_keys[0]					,  '</s>'					)
						self.assertEqual(vocab_keys[1]					,  '<unk>'					)
						self.assertEqual(vocab_keys[-1]					,  '<pad>'					)
						self.assertEqual(len(__snake_case					)					,  9					)





			def 		__magic_name__    (		self					)   -> str:




						'''simple docstring'''
						self.assertEqual(self.get_tokenizer().vocab_size					,  9					)





			def 		__magic_name__    (		self					)   -> Optional[int]:




						'''simple docstring'''
						__a        =MarianTokenizer.from_pretrained(f'{ORG_NAME}opus-mt-en-de'					)
						__a        =en_de_tokenizer(['I am a small frog']					,  return_tensors=__snake_case					)
						self.assertIsInstance(__snake_case					,  __snake_case					)
						__a        =[38, 121, 14, 697, 3_8848, 0]
						self.assertListEqual(__snake_case					,  batch.input_ids[0]					)

						__a        =tempfile.mkdtemp()
						en_de_tokenizer.save_pretrained(__snake_case					)
						__a        =[x.name for x in Path(__snake_case					).glob('*'					)]
						self.assertIn('source.spm'					,  __snake_case					)
						MarianTokenizer.from_pretrained(__snake_case					)





			def 		__magic_name__    (		self					)   -> Any:




						'''simple docstring'''
						__a        =self.get_tokenizer()

						__a        =tok(
						    ['I am a small frog' * 1000, 'I am a small frog']					,  padding=__snake_case					,  truncation=__snake_case					,  return_tensors=__snake_case					)
						self.assertIsInstance(__snake_case					,  __snake_case					)
						self.assertEqual(batch.input_ids.shape					,  (2, 512)					)





			def 		__magic_name__    (		self					)   -> Optional[Any]:




						'''simple docstring'''
						__a        =self.get_tokenizer()
						__a        =tok(['I am a tiny frog', 'I am a small frog']					,  padding=__snake_case					,  return_tensors=__snake_case					)
						self.assertIsInstance(__snake_case					,  __snake_case					)
						self.assertEqual(batch_smaller.input_ids.shape					,  (2, 10)					)





			@slow
			def 		__magic_name__    (		self					)   -> Union[str, Any]:




						'''simple docstring'''
						# fmt: off
						__a        ={'input_ids': [[4_3495, 462, 20, 4_2164, 1369, 52, 464, 132, 1703, 492, 13, 7491, 3_8999, 6, 8, 464, 132, 1703, 492, 13, 4669, 3_7867, 13, 7525, 27, 1593, 988, 13, 3_3972, 7029, 6, 20, 8251, 383, 2, 270, 5866, 3788, 2, 2353, 8251, 1_2338, 2, 1_3958, 387, 2, 3629, 6953, 188, 2900, 2, 1_3958, 8011, 1_1501, 23, 8460, 4073, 3_4009, 20, 435, 1_1439, 27, 8, 8460, 4073, 6004, 20, 9988, 375, 27, 33, 266, 1945, 1076, 1350, 3_7867, 3288, 5, 577, 1076, 4374, 8, 5082, 5, 2_6453, 257, 556, 403, 2, 242, 132, 383, 316, 492, 8, 1_0767, 6, 316, 304, 4239, 3, 0], [148, 1_5722, 19, 1839, 12, 1350, 13, 2_2327, 5082, 5418, 4_7567, 3_5938, 59, 318, 1_9552, 108, 2183, 54, 1_4976, 4835, 32, 547, 1114, 8, 315, 2417, 5, 92, 1_9088, 3, 0, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100], [36, 6395, 1_2570, 3_9147, 1_1597, 6, 266, 4, 4_5405, 7296, 3, 0, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100, 5_8100]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
						# fmt: on

						self.tokenizer_integration_test_util(
						    expected_encoding=__snake_case					,  model_name='Helsinki-NLP/opus-mt-en-de'					,  revision='1a8c2263da11e68e50938f97e10cd57820bd504c'					,  decode_kwargs={'use_source_tokenizer': True}					,  )





			def 		__magic_name__    (		self					)   -> Dict:




						'''simple docstring'''
						__a        =MarianTokenizer.from_pretrained('hf-internal-testing/test-marian-two-vocabs'					)

						__a        ='Tämä on testi'
						__a        ='This is a test'

						__a        =[76, 7, 2047, 2]
						__a        =[69, 12, 11, 940, 2]

						__a        =tokenizer(__snake_case					).input_ids
						self.assertListEqual(__snake_case					,  __snake_case					)

						__a        =tokenizer(text_target=__snake_case					).input_ids
						self.assertListEqual(__snake_case					,  __snake_case					)

						__a        =tokenizer.decode(__snake_case					,  skip_special_tokens=__snake_case					)
						self.assertEqual(__snake_case					,  __snake_case					)









import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCAmelCase					:			List[Any]  				=			get_tests_dir("fixtures/test_sentencepiece_with_bytefallback.model")






@require_sentencepiece
@require_tokenizers
class 	__magic_name__    (			lowerCAmelCase_  ,    unittest.TestCase      ):
			SCREAMING_SNAKE_CASE							   =						GPTSwaTokenizer
			SCREAMING_SNAKE_CASE							   =						False
			SCREAMING_SNAKE_CASE							   =						True
			SCREAMING_SNAKE_CASE							   =						False





			def 		__magic_name__    (		self					)   -> Dict:




						'''simple docstring'''
						super().setUp()

						# We have a SentencePiece fixture for testing
						__a        =GPTSwaTokenizer(__snake_case					,  eos_token='<unk>'					,  bos_token='<unk>'					,  pad_token='<unk>'					)

						tokenizer.save_pretrained(self.tmpdirname					)





			def 		__magic_name__    (		self					,  __snake_case					)   -> Tuple:




						'''simple docstring'''
						__a        ='This is a test'
						__a        ='This is a test'
						return input_text, output_text





			def 		__magic_name__    (		self					)   -> Optional[Any]:




						'''simple docstring'''
						__a        ='<s>'
						__a        =1

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case					)					,  __snake_case					)
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case					)					,  __snake_case					)





			def 		__magic_name__    (		self					)   -> Dict:




						'''simple docstring'''
						__a        =list(self.get_tokenizer().get_vocab().keys()					)

						self.assertEqual(vocab_keys[0]					,  '<unk>'					)
						self.assertEqual(vocab_keys[1]					,  '<s>'					)
						self.assertEqual(vocab_keys[-1]					,  'j'					)
						self.assertEqual(len(__snake_case					)					,  2000					)





			def 		__magic_name__    (		self					)   -> Optional[Any]:




						'''simple docstring'''
						self.assertEqual(self.get_tokenizer().vocab_size					,  2000					)





			def 		__magic_name__    (		self					)   -> Any:




						'''simple docstring'''
						__a        =GPTSwaTokenizer(__snake_case					)

						__a        =tokenizer.tokenize('This is a test'					)
						self.assertListEqual(__snake_case					,  ['▁This', '▁is', '▁a', '▁t', 'est']					)

						self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case					)					,  [465, 287, 265, 631, 842]					)

						__a        =tokenizer.tokenize('I was born in 92000, and this is falsé.'					)
						# fmt: off
						self.assertListEqual(
						    __snake_case					,  ['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']					,  )
						# fmt: on

						__a        =tokenizer.convert_tokens_to_ids(__snake_case					)
						self.assertListEqual(
						    __snake_case					,  [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]					,  )

						__a        =tokenizer.convert_ids_to_tokens(__snake_case					)
						# fmt: off
						self.assertListEqual(
						    __snake_case					,  ['▁I', '▁was', '▁bor', 'n', '▁in', '▁', '<0x39>', '2', '0', '0', '0', ',', '▁and', '▁this', '▁is', '▁f', 'al', 's', '<0xC3>', '<0xA9>', '.']					)
						# fmt: on





			def 		__magic_name__    (		self					)   -> Optional[int]:




						'''simple docstring'''
						__a        =GPTSwaTokenizer(__snake_case					)
						__a        =['This is a test', 'I was born in 92000, and this is falsé.']
						__a        =[
						    [465, 287, 265, 631, 842],
						    [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
						]

						# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
						for text, expected_ids in zip(__snake_case					,  __snake_case					):
									self.assertListEqual(tokenizer.encode_fast(__snake_case					)					,  __snake_case					)

						# Test that decode_fast returns the input text
						for text, token_ids in zip(__snake_case					,  __snake_case					):
									self.assertEqual(tokenizer.decode_fast(__snake_case					)					,  __snake_case					)





			@slow
			def 		__magic_name__    (		self					)   -> List[str]:




						'''simple docstring'''
						__a        =[
						    '<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')',
						    'Hey there, how are you doing this fine day?',
						    'This is a text with a trailing spaces followed by a dot     .',
						    'Häj sväjs lillebrör! =)',
						    'Det är inget fel på Mr. Cool',
						]

						# fmt: off
						__a        ={'input_ids': [[6_3423, 5, 6811, 1_4954, 282, 816, 3821, 6_3466, 6_3425, 6_3462, 18, 6_3978, 678, 301, 1320, 6_3423, 6_3455, 6_3458, 18, 6_3982, 4246, 3940, 1901, 4_7789, 5547, 1_8994], [1_9630, 1100, 6_3446, 1342, 633, 544, 4488, 593, 5102, 2416, 6_3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 5_8593, 2_2413, 9106, 546, 268, 3_3213, 6_3979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5130, 6_3450, 924, 6_3449, 2249, 4062, 1558, 318, 6_3504, 2_1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 6_3443, 2_6801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
						# fmt: on
						self.tokenizer_integration_test_util(
						    expected_encoding=__snake_case					,  model_name='AI-Sweden/gpt-sw3-126m'					,  sequences=__snake_case					,  )






