


import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
						from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
						__lowerCAmelCase	: Tuple  				=get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
						from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__lowerCAmelCase	: Any  				=1_2_8_0_2_2
__lowerCAmelCase	: List[Any]  				=1_2_8_0_2_8
@require_sentencepiece
class        _lowercase      (			A__						,			unittest.TestCase       ):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__				:			Tuple						=		MaMaaaTokenizer
				SCREAMING_SNAKE_CASE__				:			List[Any]						=		False
				SCREAMING_SNAKE_CASE__				:			Dict						=		False
				SCREAMING_SNAKE_CASE__				:			Dict						=		True


				def 		__magic_name__(       self			:Dict	)		->			Any:
										super().setUp()

										__SCREAMING_SNAKE_CASE						:     int					=		['''</s>''', '''<unk>''', '''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''', '''\u0120''', '''<pad>''']
										__SCREAMING_SNAKE_CASE						:     Any					=		dict(zip(lowerCAmelCase__			,			range(len(lowerCAmelCase__	)	)	)	)
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		Path(self.tmpdirname	)
										save_json(lowerCAmelCase__			,			save_dir / VOCAB_FILES_NAMES['''vocab_file''']	)
										if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
																copyfile(lowerCAmelCase__			,			save_dir / VOCAB_FILES_NAMES['''spm_file''']	)

										__SCREAMING_SNAKE_CASE						:     List[str]					=		MaMaaaTokenizer.from_pretrained(self.tmpdirname	)
										tokenizer.save_pretrained(self.tmpdirname	)


				def 		__magic_name__(       self			:Union[str, Any]			,			**lowerCAmelCase__			:List[str]	)		->			List[str]:
										return MaMaaaTokenizer.from_pretrained(self.tmpdirname			,			**lowerCAmelCase__	)


				def 		__magic_name__(       self			:str			,			lowerCAmelCase__			:Tuple	)		->			Any:
										return (
										    "This is a test",
										    "This is a test",
										)


				def 		__magic_name__(       self			:List[str]	)		->			List[Any]:
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		'''</s>'''
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		0

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__	)			,			lowerCAmelCase__	)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__	)			,			lowerCAmelCase__	)


				def 		__magic_name__(       self			:Tuple	)		->			Tuple:
										__SCREAMING_SNAKE_CASE						:     List[Any]					=		self.get_tokenizer()
										__SCREAMING_SNAKE_CASE						:     Any					=		list(tokenizer.get_vocab().keys()	)

										self.assertEqual(vocab_keys[0]			,			'''</s>'''	)
										self.assertEqual(vocab_keys[1]			,			'''<unk>'''	)
										self.assertEqual(vocab_keys[-1]			,			'''<s>'''	)
										self.assertEqual(len(lowerCAmelCase__	)			,			tokenizer.vocab_size + len(tokenizer.get_added_vocab()	)	)


				@unittest.skip('''Skip this test while all models are still to be uploaded.'''	)
				def 		__magic_name__(       self			:Optional[Any]	)		->			int:
										pass


				def 		__magic_name__(       self			:int	)		->			List[Any]:
										__SCREAMING_SNAKE_CASE						:     Tuple					=		self.get_tokenizer()

										__SCREAMING_SNAKE_CASE						:     List[str]					=		tokenizer.tokenize('''This is a test'''	)
										self.assertListEqual(lowerCAmelCase__			,			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']	)

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(lowerCAmelCase__	)			,			[2, 3, 4, 5, 6]			,			)

										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6]	)
										self.assertListEqual(lowerCAmelCase__			,			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']	)

										__SCREAMING_SNAKE_CASE						:     int					=		tokenizer.convert_tokens_to_string(lowerCAmelCase__	)
										self.assertEqual(lowerCAmelCase__			,			'''This is a test'''	)



				@slow
				def 		__magic_name__(       self			:Optional[int]	)		->			List[Any]:
										# fmt: off
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		{'''input_ids''': [[128_022, 110_108, 397, 11, 38_272, 2_247, 124_811, 285, 18_105, 1_586, 207, 7, 39_534, 4_428, 397, 1_019, 18_105, 1_586, 207, 7, 41_337, 16_786, 241, 7, 20_214, 17, 125_690, 10_398, 7, 44_378, 58_069, 68_342, 7_798, 7_343, 11, 299, 33_310, 4, 158, 37_350, 94_077, 4_569, 299, 33_310, 90, 4, 52_840, 290, 4, 31_270, 112, 299, 682, 4, 52_840, 39_953, 14_079, 193, 52_519, 90_894, 17_894, 120_697, 11, 40_445, 551, 17, 1_019, 52_519, 90_894, 17_756, 963, 11, 40_445, 480, 17, 9_792, 1_120, 5_173, 1_393, 6_240, 16_786, 241, 120_996, 28, 1_245, 1_393, 118_240, 11_123, 1_019, 93_612, 2_691, 10_618, 98_058, 120_409, 1_928, 279, 4, 40_683, 367, 178, 207, 1_019, 103, 103_121, 506, 65_296, 5, 2], [128_022, 21_217, 367, 117, 125_450, 128, 719, 7, 7_308, 40, 93_612, 12_669, 1_116, 16_704, 71, 17_785, 3_699, 15_592, 35, 144, 9_584, 241, 11_943, 713, 950, 799, 2_247, 88_427, 150, 149, 118_813, 120_706, 1_019, 106_906, 81_518, 28, 1_224, 22_799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [128_022, 1_658, 123_311, 5_155, 5_578, 4_722, 279, 14_947, 2_366, 1_120, 1_197, 14, 1_348, 9_232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=lowerCAmelCase__			,			model_name='''facebook/m2m100_418M'''			,			revision='''c168bae485c864188cf9aa0e4108b0b6934dc91e'''			,			)




@require_torch
@require_sentencepiece
@require_tokenizers
class        _lowercase      (			unittest.TestCase       ):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__				:			Optional[Any]						=		'''facebook/m2m100_418M'''
				SCREAMING_SNAKE_CASE__				:			str						=		[
				    '''In my opinion, there are two levels of response from the French government.''',
				    '''NSA Affair Emphasizes Complete Lack of Debate on Intelligence''',
				]
				SCREAMING_SNAKE_CASE__				:			Dict						=		[
				    '''Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.''',
				    '''L\'affaire NSA souligne l\'absence totale de débat sur le renseignement''',
				]

				# fmt: off
				SCREAMING_SNAKE_CASE__				:			int						=		[EN_CODE, 593, 1_949, 115_781, 4, 71_586, 4_234, 60_633, 126_233, 432, 123_808, 15_592, 1_197, 117_132, 120_618, 5, 2]


				@classmethod
				def 		__magic_name__(       cls			:Dict	)		->			int:
										__SCREAMING_SNAKE_CASE						:     MaMaaaTokenizer					=		MaMaaaTokenizer.from_pretrained(
										    cls.checkpoint_name			,			src_lang='''en'''			,			tgt_lang='''fr'''	)
										__SCREAMING_SNAKE_CASE						:     Dict					=		1
										return cls


				def 		__magic_name__(       self			:int	)		->			Dict:
										self.assertEqual(self.tokenizer.get_lang_id('''ar'''	)			,			128_006	)
										self.assertEqual(self.tokenizer.get_lang_id('''en'''	)			,			128_022	)
										self.assertEqual(self.tokenizer.get_lang_id('''ro'''	)			,			128_076	)
										self.assertEqual(self.tokenizer.get_lang_id('''mr'''	)			,			128_063	)


				def 		__magic_name__(       self			:Dict	)		->			Any:
										__SCREAMING_SNAKE_CASE						:     Any					=		self.tokenizer.get_vocab()
										self.assertEqual(len(lowerCAmelCase__	)			,			self.tokenizer.vocab_size	)
										self.assertEqual(vocab['''<unk>''']			,			3	)
										self.assertIn(self.tokenizer.get_lang_token('''en'''	)			,			lowerCAmelCase__	)


				def 		__magic_name__(       self			:str	)		->			Tuple:
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		'''en'''
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		self.tokenizer.batch_encode_plus(self.src_text	).input_ids[0]
										self.assertListEqual(self.expected_src_tokens			,			lowerCAmelCase__	)


				def 		__magic_name__(       self			:Any	)		->			Optional[int]:
										self.assertIn(lowerCAmelCase__			,			self.tokenizer.all_special_ids	)
										# fmt: off
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		[FR_CODE, 5_364, 82, 8_642, 4, 294, 47, 8, 14_028, 136, 3_286, 9_706, 6, 90_797, 6, 144_012, 162, 88_128, 30_061, 5, 2]
										# fmt: on
										__SCREAMING_SNAKE_CASE						:     Tuple					=		self.tokenizer.decode(lowerCAmelCase__			,			skip_special_tokens=lowerCAmelCase__	)
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		self.tokenizer.decode(generated_ids[1:]			,			skip_special_tokens=lowerCAmelCase__	)
										self.assertEqual(lowerCAmelCase__			,			lowerCAmelCase__	)
										self.assertNotIn(self.tokenizer.eos_token			,			lowerCAmelCase__	)


				def 		__magic_name__(       self			:str	)		->			int:
										__SCREAMING_SNAKE_CASE						:     Dict					=		tempfile.mkdtemp()
										__SCREAMING_SNAKE_CASE						:     List[str]					=		self.tokenizer.lang_token_to_id
										self.tokenizer.save_pretrained(lowerCAmelCase__	)
										__SCREAMING_SNAKE_CASE						:     Dict					=		MaMaaaTokenizer.from_pretrained(lowerCAmelCase__	)
										self.assertDictEqual(new_tok.lang_token_to_id			,			lowerCAmelCase__	)


				@require_torch
				def 		__magic_name__(       self			:Optional[Any]	)		->			Tuple:
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		'''en'''
										__SCREAMING_SNAKE_CASE						:     Dict					=		'''fr'''

										__SCREAMING_SNAKE_CASE						:     List[str]					=		self.tokenizer(self.src_text			,			text_target=self.tgt_text			,			padding=lowerCAmelCase__			,			return_tensors='''pt'''	)

										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		shift_tokens_right(
										    batch['''labels''']			,			self.tokenizer.pad_token_id			,			self.tokenizer.eos_token_id	)

										for k in batch:
																__SCREAMING_SNAKE_CASE						:     List[str]					=		batch[k].tolist()
										# batch = {k: v.tolist() for k,v in batch.items()}
										# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
										# batch.decoder_inputs_ids[0][0] ==
										assert batch.input_ids[1][0] == EN_CODE
										assert batch.input_ids[1][-1] == 2
										assert batch.labels[1][0] == FR_CODE
										assert batch.labels[1][-1] == 2
										assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]


				@require_torch
				def 		__magic_name__(       self			:Dict	)		->			Optional[Any]:
										__SCREAMING_SNAKE_CASE						:     Dict					=		'''mr'''
										self.assertListEqual(self.tokenizer.prefix_tokens			,			[self.tokenizer.get_lang_id('''mr'''	)]	)
										self.assertListEqual(self.tokenizer.suffix_tokens			,			[self.tokenizer.eos_token_id]	)

										__SCREAMING_SNAKE_CASE						:     int					=		'''zh'''
										self.assertListEqual(self.tokenizer.prefix_tokens			,			[self.tokenizer.get_lang_id('''zh'''	)]	)
										self.assertListEqual(self.tokenizer.suffix_tokens			,			[self.tokenizer.eos_token_id]	)


				@require_torch
				def 		__magic_name__(       self			:List[Any]	)		->			Optional[Any]:
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		'''mr'''
										self.tokenizer._switch_to_target_mode()
										self.assertListEqual(self.tokenizer.prefix_tokens			,			[self.tokenizer.get_lang_id('''mr'''	)]	)
										self.assertListEqual(self.tokenizer.suffix_tokens			,			[self.tokenizer.eos_token_id]	)
										self.tokenizer._switch_to_input_mode()
										self.assertListEqual(self.tokenizer.prefix_tokens			,			[self.tokenizer.get_lang_id(self.tokenizer.src_lang	)]	)

										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		'''zh'''
										self.tokenizer._switch_to_target_mode()
										self.assertListEqual(self.tokenizer.prefix_tokens			,			[self.tokenizer.get_lang_id('''zh'''	)]	)
										self.assertListEqual(self.tokenizer.suffix_tokens			,			[self.tokenizer.eos_token_id]	)
										self.tokenizer._switch_to_input_mode()
										self.assertListEqual(self.tokenizer.prefix_tokens			,			[self.tokenizer.get_lang_id(self.tokenizer.src_lang	)]	)



				@require_torch
				def 		__magic_name__(       self			:Tuple	)		->			str:
										__SCREAMING_SNAKE_CASE						:     Dict					=		self.tokenizer._build_translation_inputs('''A test'''			,			return_tensors='''pt'''			,			src_lang='''en'''			,			tgt_lang='''ar'''	)

										self.assertEqual(
										    nested_simplify(lowerCAmelCase__	)			,			{
										        # en_XX, A, test, EOS
										        '''input_ids''': [[128_022, 58, 4_183, 2]],
										        '''attention_mask''': [[1, 1, 1, 1]],
										        # ar_AR
										        '''forced_bos_token_id''': 128_006,
										    }			,			)




'''simple docstring'''



A__		:					Any												=  8.3_1_4_4_6_2  # Unit - J mol-1 K-1



def 			UpperCAmelCase__					(	UpperCAmelCase_    :				float      ,    UpperCAmelCase_    :				float      ,    UpperCAmelCase_    :				float							)     ->		float:
     if moles < 0 or kelvin < 0 or volume < 0:
          raise ValueError('Invalid inputs. Enter positive value.'							)
     return moles * kelvin * UNIVERSAL_GAS_CONSTANT / volume



def 			UpperCAmelCase__					(	UpperCAmelCase_    :				float      ,    UpperCAmelCase_    :				float      ,    UpperCAmelCase_    :				float							)     ->		float:
     if moles < 0 or kelvin < 0 or pressure < 0:
          raise ValueError('Invalid inputs. Enter positive value.'							)
     return moles * kelvin * UNIVERSAL_GAS_CONSTANT / pressure


if __name__ == "__main__":
      from doctest import testmod

      testmod()






