




'''simple docstring'''



import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_       =							get_tests_dir("fixtures/test_sentencepiece.model")

if is_sentencepiece_available():
      import sentencepiece as sp


UpperCamelCase_       =							5
UpperCamelCase_       =							1_0






@require_sentencepiece
@require_tokenizers
class 				_a							(       UpperCAmelCase__      ,	unittest.TestCase      ):





       '''simple docstring'''






       A		:							Tuple								=		SpeechaTextTokenizer
       A		:							int								=		False
       A		:							Union[str, Any]								=		True







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             super().setUp()

             SCREAMING_SNAKE_CASE					:	List[Any]   =		sp.SentencePieceProcessor()
             spm_model.Load(lowerCamelCase__   )
             SCREAMING_SNAKE_CASE					:	Optional[int]   =		["<s>", "<pad>", "</s>", "<unk>"]

             vocab += [spm_model.IdToPiece(id_   ) for id_ in range(len(lowerCamelCase__   )   )]
             SCREAMING_SNAKE_CASE					:	Any   =		dict(zip(lowerCamelCase__,				range(len(lowerCamelCase__   )   )   )   )

             SCREAMING_SNAKE_CASE					:	Union[str, Any]   =		Path(self.tmpdirname   )
             save_json(lowerCamelCase__,				save_dir / VOCAB_FILES_NAMES['vocab_file']   )
             if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
                   copyfile(lowerCamelCase__,				save_dir / VOCAB_FILES_NAMES['spm_file']   )

             SCREAMING_SNAKE_CASE					:	Tuple   =		SpeechaTextTokenizer.from_pretrained(self.tmpdirname   )
             tokenizer.save_pretrained(self.tmpdirname   )







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	Dict   =		"<pad>"
             SCREAMING_SNAKE_CASE					:	Any   =		1

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase__   ),				lowerCamelCase__   )
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase__   ),				lowerCamelCase__   )







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	str   =		list(self.get_tokenizer().get_vocab().keys()   )

             self.assertEqual(vocab_keys[0],				'<s>'   )
             self.assertEqual(vocab_keys[1],				'<pad>'   )
             self.assertEqual(vocab_keys[-1],				'j'   )
             self.assertEqual(len(lowerCamelCase__   ),				1_001   )







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             self.assertEqual(self.get_tokenizer().vocab_size,				1_001   )







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	Dict   =		SpeechaTextTokenizer.from_pretrained(self.tmpdirname   )

             SCREAMING_SNAKE_CASE					:	Any   =		tokenizer.tokenize('This is a test'   )
             self.assertListEqual(lowerCamelCase__,				['▁This', '▁is', '▁a', '▁t', 'est']   )

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(lowerCamelCase__   ),				[289, 50, 14, 174, 386],				)

             SCREAMING_SNAKE_CASE					:	Optional[Any]   =		tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
             self.assertListEqual(
                 lowerCamelCase__,				[SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '9', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', 'é', '.'],				)
             SCREAMING_SNAKE_CASE					:	Any   =		tokenizer.convert_tokens_to_ids(lowerCamelCase__   )
             self.assertListEqual(lowerCamelCase__,				[12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8]   )

             SCREAMING_SNAKE_CASE					:	str   =		tokenizer.convert_ids_to_tokens(lowerCamelCase__   )
             self.assertListEqual(
                 lowerCamelCase__,				[SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '<unk>', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', '<unk>', '.'],				)







       @slow
       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	List[str]   =		{"input_ids": [[3_791, 797, 31, 11, 64, 797, 31, 2_429, 433, 12, 1_176, 12, 20, 786, 915, 142, 2_413, 240, 37, 3_238, 797, 31, 11, 35, 93, 915, 142, 2_413, 240, 37, 5_540, 567, 1_276, 93, 37, 610, 40, 62, 455, 657, 1_042, 123, 780, 177, 37, 309, 241, 1_298, 514, 20, 292, 2_737, 114, 2_469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3_388, 511, 459, 4, 3_555, 40, 321, 302, 705, 4, 3_388, 511, 583, 326, 5, 5, 5, 62, 3_310, 560, 177, 2_680, 217, 1_508, 32, 31, 853, 418, 64, 583, 511, 1_605, 62, 35, 93, 560, 177, 2_680, 217, 1_508, 1_521, 64, 583, 511, 519, 62, 20, 1_515, 764, 20, 149, 261, 5_625, 7_972, 20, 5_540, 567, 1_276, 93, 3_925, 1_675, 11, 15, 802, 7_972, 576, 217, 1_508, 11, 35, 93, 1_253, 2_441, 15, 289, 652, 31, 416, 321, 3_842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2_681, 1_153, 3_434, 20, 5_540, 37, 567, 126, 1_253, 2_441, 3_376, 449, 210, 431, 1_563, 177, 767, 5_540, 11, 1_203, 472, 11, 2_953, 685, 285, 364, 706, 1_153, 20, 6_799, 20, 2_869, 20, 4_464, 126, 40, 2_429, 20, 1_040, 866, 2_664, 418, 20, 318, 20, 1_726, 186, 20, 265, 522, 35, 93, 2_191, 4_634, 20, 1_040, 12, 6_799, 15, 228, 2_356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_575, 2_666, 684, 1_582, 1_176, 12, 627, 149, 619, 20, 4_902, 563, 11, 20, 149, 261, 3_420, 2_356, 174, 142, 4_714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=lowerCamelCase__,				model_name='facebook/s2t-small-mustc-en-de-st',				revision='a14f04cf0776c02f62a8cb800cf7909e15ea23ad',				)






@require_sentencepiece
class 				_a							(       unittest.TestCase      ):





       '''simple docstring'''






       A		:							List[str]								=		"valhalla/s2t_mustc_multilinguial_medium"

       A		:							List[Any]								=		"C'est trop cool"
       A		:							Dict								=		"Esto es genial"







       @classmethod
       def     UpperCamelCase_				(	cls   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	SpeechaTextTokenizer   =		SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name   )
             return cls







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             self.assertEqual(self.tokenizer.lang_code_to_id['pt'],				4   )
             self.assertEqual(self.tokenizer.lang_code_to_id['ru'],				6   )
             self.assertEqual(self.tokenizer.lang_code_to_id['it'],				9   )
             self.assertEqual(self.tokenizer.lang_code_to_id['de'],				11   )







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             self.assertEqual(self.tokenizer.vocab_size,				10_000   )







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             self.assertIn(lowerCamelCase__,				self.tokenizer.all_special_ids   )
             SCREAMING_SNAKE_CASE					:	List[Any]   =		[ES_CODE, 4, 1_601, 47, 7_647, 2]
             SCREAMING_SNAKE_CASE					:	str   =		self.tokenizer.decode(lowerCamelCase__,				skip_special_tokens=lowerCamelCase__   )
             SCREAMING_SNAKE_CASE					:	Optional[int]   =		self.tokenizer.decode(generated_ids[1:],				skip_special_tokens=lowerCamelCase__   )
             self.assertEqual(lowerCamelCase__,				lowerCamelCase__   )
             self.assertNotIn(self.tokenizer.eos_token,				lowerCamelCase__   )







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	Dict   =		"fr"
             SCREAMING_SNAKE_CASE					:	str   =		self.tokenizer(self.french_text   ).input_ids
             self.assertEqual(encoded[0],				lowerCamelCase__   )
             self.assertEqual(encoded[-1],				self.tokenizer.eos_token_id   )







       def     UpperCamelCase_				(	self   ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE					:	Dict   =		"fr"
             self.assertListEqual(self.tokenizer.prefix_tokens,				[FR_CODE]   )

             SCREAMING_SNAKE_CASE					:	Tuple   =		"es"
             self.assertListEqual(self.tokenizer.prefix_tokens,				[ES_CODE]   )






'''simple docstring'''


import os
import tempfile
import unittest

from transformers.models.marian.convert_marian_tatoeba_to_pytorch import DEFAULT_REPO, TatoebaConverter
from transformers.testing_utils import slow
from transformers.utils import cached_property






@unittest.skipUnless(os.path.exists(UpperCAmelCase__		)			,				"Tatoeba directory does not exist."		)
class 		lowerCAmelCase__	(	unittest.TestCase		):




			@cached_property
			def 							lowerCAmelCase__			(						self     :   Optional[Any]       )      ->List[Any]:



					'''simple docstring'''







					_UpperCAmelCase						:				List[str]	    =					tempfile.mkdtemp()
					return TatoebaConverter(save_dir=lowerCamelCase__       )




			@slow
			def 							lowerCAmelCase__			(						self     :   Optional[int]       )      ->List[str]:



					'''simple docstring'''







					self.resolver.convert_models(["heb-eng"]       )






			@slow
			def 							lowerCAmelCase__			(						self     :   List[Any]       )      ->Any:



					'''simple docstring'''







					_UpperCAmelCase    ,			_UpperCAmelCase						:				Tuple	    =					self.resolver.write_model_card("opus-mt-he-en"		,     dry_run=lowerCamelCase__       )
					assert mmeta["long_pair"] == "heb-eng"



