import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
    from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
    _SCREAMING_SNAKE_CASE										=						get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
    from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

_SCREAMING_SNAKE_CASE										=						128_022
_SCREAMING_SNAKE_CASE										=						128_028



@require_sentencepiece
class 				a							(			__lowerCAmelCase       ,			unittest.TestCase       ):







 """simple docstring"""






 lowerCamelCase	:Tuple    			=		MaMaaaTokenizer
 lowerCamelCase	:List[Any]    			=		False
 lowerCamelCase	:Optional[int]    			=		False
 lowerCamelCase	:int    			=		True



 def      UpperCAmelCase							(					self							)  ->					Optional[Any]:
        super().setUp()

        _A        =  ["""</s>""", """<unk>""", """▁This""", """▁is""", """▁a""", """▁t""", """est""", """\u0120""", """<pad>"""]
        _A        =  dict(zip(lowerCAmelCase_		,				range(len(lowerCAmelCase_							)							)							)							)
        _A        =  Path(self.tmpdirname							)
        save_json(lowerCAmelCase_		,				save_dir / VOCAB_FILES_NAMES["""vocab_file"""]							)
        if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
               copyfile(lowerCAmelCase_		,				save_dir / VOCAB_FILES_NAMES["""spm_file"""]							)

        _A        =  MaMaaaTokenizer.from_pretrained(self.tmpdirname							)
        tokenizer.save_pretrained(self.tmpdirname							)



 def      UpperCAmelCase							(					self		,				**lowerCAmelCase_							)  ->					List[Any]:
        return MaMaaaTokenizer.from_pretrained(self.tmpdirname		,				**lowerCAmelCase_							)



 def      UpperCAmelCase							(					self		,				lowerCAmelCase_							)  ->					Any:
        return (
            "This is a test",
            "This is a test",
        )



 def      UpperCAmelCase							(					self							)  ->					Tuple:
        _A        =  """</s>"""
        _A        =  0

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase_							)		,				lowerCAmelCase_							)
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase_							)		,				lowerCAmelCase_							)



 def      UpperCAmelCase							(					self							)  ->					List[Any]:
        _A        =  self.get_tokenizer()
        _A        =  list(tokenizer.get_vocab().keys()							)

        self.assertEqual(vocab_keys[0]		,				"""</s>"""							)
        self.assertEqual(vocab_keys[1]		,				"""<unk>"""							)
        self.assertEqual(vocab_keys[-1]		,				"""<s>"""							)
        self.assertEqual(len(lowerCAmelCase_							)		,				tokenizer.vocab_size + len(tokenizer.get_added_vocab()							)							)



 @unittest.skip("""Skip this test while all models are still to be uploaded."""							)
 def      UpperCAmelCase							(					self							)  ->					Tuple:
        pass



 def      UpperCAmelCase							(					self							)  ->					Tuple:
        _A        =  self.get_tokenizer()

        _A        =  tokenizer.tokenize("""This is a test"""							)
        self.assertListEqual(lowerCAmelCase_		,				["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]							)

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(lowerCAmelCase_							)		,				[2, 3, 4, 5, 6]		,				)

        _A        =  tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6]							)
        self.assertListEqual(lowerCAmelCase_		,				["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]							)

        _A        =  tokenizer.convert_tokens_to_string(lowerCAmelCase_							)
        self.assertEqual(lowerCAmelCase_		,				"""This is a test"""							)



 @slow
 def      UpperCAmelCase							(					self							)  ->					str:
        # fmt: off
        _A        =  {"""input_ids""": [[12_80_22, 11_01_08, 3_97, 11, 3_82_72, 22_47, 12_48_11, 2_85, 1_81_05, 15_86, 2_07, 7, 3_95_34, 44_28, 3_97, 10_19, 1_81_05, 15_86, 2_07, 7, 4_13_37, 1_67_86, 2_41, 7, 2_02_14, 17, 12_56_90, 1_03_98, 7, 4_43_78, 5_80_69, 6_83_42, 77_98, 73_43, 11, 2_99, 3_33_10, 4, 1_58, 3_73_50, 9_40_77, 45_69, 2_99, 3_33_10, 90, 4, 5_28_40, 2_90, 4, 3_12_70, 1_12, 2_99, 6_82, 4, 5_28_40, 3_99_53, 1_40_79, 1_93, 5_25_19, 9_08_94, 1_78_94, 12_06_97, 11, 4_04_45, 5_51, 17, 10_19, 5_25_19, 9_08_94, 1_77_56, 9_63, 11, 4_04_45, 4_80, 17, 97_92, 11_20, 51_73, 13_93, 62_40, 1_67_86, 2_41, 12_09_96, 28, 12_45, 13_93, 11_82_40, 1_11_23, 10_19, 9_36_12, 26_91, 1_06_18, 9_80_58, 12_04_09, 19_28, 2_79, 4, 4_06_83, 3_67, 1_78, 2_07, 10_19, 1_03, 10_31_21, 5_06, 6_52_96, 5, 2], [12_80_22, 2_12_17, 3_67, 1_17, 12_54_50, 1_28, 7_19, 7, 73_08, 40, 9_36_12, 1_26_69, 11_16, 1_67_04, 71, 1_77_85, 36_99, 1_55_92, 35, 1_44, 95_84, 2_41, 1_19_43, 7_13, 9_50, 7_99, 22_47, 8_84_27, 1_50, 1_49, 11_88_13, 12_07_06, 10_19, 10_69_06, 8_15_18, 28, 12_24, 2_27_99, 3_97, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [12_80_22, 16_58, 12_33_11, 51_55, 55_78, 47_22, 2_79, 1_49_47, 23_66, 11_20, 11_97, 14, 13_48, 92_32, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=lowerCAmelCase_		,				model_name="""facebook/m2m100_418M"""		,				revision="""c168bae485c864188cf9aa0e4108b0b6934dc91e"""		,				)



@require_torch
@require_sentencepiece
@require_tokenizers
class 				a							(			unittest.TestCase       ):







 """simple docstring"""






 lowerCamelCase	:Union[str, Any]    			=		'''facebook/m2m100_418M'''
 lowerCamelCase	:Optional[int]    			=		[
     '''In my opinion, there are two levels of response from the French government.''',
     '''NSA Affair Emphasizes Complete Lack of Debate on Intelligence''',
 ]
 lowerCamelCase	:str    			=		[
     '''Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.''',
     '''L\'affaire NSA souligne l\'absence totale de débat sur le renseignement''',
 ]

 # fmt: off
 lowerCamelCase	:Tuple    			=		[EN_CODE, 593, 1949, 115781, 4, 71586, 4234, 60633, 126233, 432, 123808, 15592, 1197, 117132, 120618, 5, 2]



 @classmethod
 def      UpperCAmelCase							(					cls							)  ->					Any:
        _A        =  MaMaaaTokenizer.from_pretrained(
            cls.checkpoint_name		,				src_lang="""en"""		,				tgt_lang="""fr"""							)
        _A        =  1
        return cls



 def      UpperCAmelCase							(					self							)  ->					List[Any]:
        self.assertEqual(self.tokenizer.get_lang_id("""ar"""							)		,				12_80_06							)
        self.assertEqual(self.tokenizer.get_lang_id("""en"""							)		,				12_80_22							)
        self.assertEqual(self.tokenizer.get_lang_id("""ro"""							)		,				12_80_76							)
        self.assertEqual(self.tokenizer.get_lang_id("""mr"""							)		,				12_80_63							)



 def      UpperCAmelCase							(					self							)  ->					List[str]:
        _A        =  self.tokenizer.get_vocab()
        self.assertEqual(len(lowerCAmelCase_							)		,				self.tokenizer.vocab_size							)
        self.assertEqual(vocab["""<unk>"""]		,				3							)
        self.assertIn(self.tokenizer.get_lang_token("""en"""							)		,				lowerCAmelCase_							)



 def      UpperCAmelCase							(					self							)  ->					Optional[Any]:
        _A        =  """en"""
        _A        =  self.tokenizer.batch_encode_plus(self.src_text							).input_ids[0]
        self.assertListEqual(self.expected_src_tokens		,				lowerCAmelCase_							)



 def      UpperCAmelCase							(					self							)  ->					Dict:
        self.assertIn(lowerCAmelCase_		,				self.tokenizer.all_special_ids							)
        # fmt: off
        _A        =  [FR_CODE, 53_64, 82, 86_42, 4, 2_94, 47, 8, 1_40_28, 1_36, 32_86, 97_06, 6, 9_07_97, 6, 14_40_12, 1_62, 8_81_28, 3_00_61, 5, 2]
        # fmt: on
        _A        =  self.tokenizer.decode(lowerCAmelCase_		,				skip_special_tokens=lowerCAmelCase_							)
        _A        =  self.tokenizer.decode(generated_ids[1:]		,				skip_special_tokens=lowerCAmelCase_							)
        self.assertEqual(lowerCAmelCase_		,				lowerCAmelCase_							)
        self.assertNotIn(self.tokenizer.eos_token		,				lowerCAmelCase_							)



 def      UpperCAmelCase							(					self							)  ->					int:
        _A        =  tempfile.mkdtemp()
        _A        =  self.tokenizer.lang_token_to_id
        self.tokenizer.save_pretrained(lowerCAmelCase_							)
        _A        =  MaMaaaTokenizer.from_pretrained(lowerCAmelCase_							)
        self.assertDictEqual(new_tok.lang_token_to_id		,				lowerCAmelCase_							)



 @require_torch
 def      UpperCAmelCase							(					self							)  ->					Union[str, Any]:
        _A        =  """en"""
        _A        =  """fr"""

        _A        =  self.tokenizer(self.src_text		,				text_target=self.tgt_text		,				padding=lowerCAmelCase_		,				return_tensors="""pt"""							)

        _A        =  shift_tokens_right(
            batch["""labels"""]		,				self.tokenizer.pad_token_id		,				self.tokenizer.eos_token_id							)

        for k in batch:
               _A        =  batch[k].tolist()
        # batch = {k: v.tolist() for k,v in batch.items()}
        # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
        # batch.decoder_inputs_ids[0][0] ==
        assert batch.input_ids[1][0] == EN_CODE
        assert batch.input_ids[1][-1] == 2
        assert batch.labels[1][0] == FR_CODE
        assert batch.labels[1][-1] == 2
        assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]



 @require_torch
 def      UpperCAmelCase							(					self							)  ->					List[str]:
        _A        =  """mr"""
        self.assertListEqual(self.tokenizer.prefix_tokens		,				[self.tokenizer.get_lang_id("""mr"""							)]							)
        self.assertListEqual(self.tokenizer.suffix_tokens		,				[self.tokenizer.eos_token_id]							)

        _A        =  """zh"""
        self.assertListEqual(self.tokenizer.prefix_tokens		,				[self.tokenizer.get_lang_id("""zh"""							)]							)
        self.assertListEqual(self.tokenizer.suffix_tokens		,				[self.tokenizer.eos_token_id]							)



 @require_torch
 def      UpperCAmelCase							(					self							)  ->					Tuple:
        _A        =  """mr"""
        self.tokenizer._switch_to_target_mode()
        self.assertListEqual(self.tokenizer.prefix_tokens		,				[self.tokenizer.get_lang_id("""mr"""							)]							)
        self.assertListEqual(self.tokenizer.suffix_tokens		,				[self.tokenizer.eos_token_id]							)
        self.tokenizer._switch_to_input_mode()
        self.assertListEqual(self.tokenizer.prefix_tokens		,				[self.tokenizer.get_lang_id(self.tokenizer.src_lang							)]							)

        _A        =  """zh"""
        self.tokenizer._switch_to_target_mode()
        self.assertListEqual(self.tokenizer.prefix_tokens		,				[self.tokenizer.get_lang_id("""zh"""							)]							)
        self.assertListEqual(self.tokenizer.suffix_tokens		,				[self.tokenizer.eos_token_id]							)
        self.tokenizer._switch_to_input_mode()
        self.assertListEqual(self.tokenizer.prefix_tokens		,				[self.tokenizer.get_lang_id(self.tokenizer.src_lang							)]							)



 @require_torch
 def      UpperCAmelCase							(					self							)  ->					Optional[int]:
        _A        =  self.tokenizer._build_translation_inputs("""A test"""		,				return_tensors="""pt"""		,				src_lang="""en"""		,				tgt_lang="""ar"""							)

        self.assertEqual(
            nested_simplify(lowerCAmelCase_							)		,				{
                # en_XX, A, test, EOS
                """input_ids""": [[12_80_22, 58, 41_83, 2]],
                """attention_mask""": [[1, 1, 1, 1]],
                # ar_AR
                """forced_bos_token_id""": 12_80_06,
            }		,				)





import unittest

from transformers import JukeboxTokenizer
from transformers.testing_utils import require_torch



class 				a							(			unittest.TestCase       ):







 """simple docstring"""






 lowerCamelCase	:Tuple    			=		JukeboxTokenizer
 lowerCamelCase	:str    			=		{
     '''artist''': '''Zac Brown Band''',
     '''genres''': '''Country''',
     '''lyrics''': '''I met a traveller from an antique land,
        Who said "Two vast and trunkless legs of stone
        Stand in the desert. . . . Near them, on the sand,
        Half sunk a shattered visage lies, whose frown,
        And wrinkled lip, and sneer of cold command,
        Tell that its sculptor well those passions read
        Which yet survive, stamped on these lifeless things,
        The hand that mocked them, and the heart that fed;
        And on the pedestal, these words appear:
        My name is Ozymandias, King of Kings;
        Look on my Works, ye Mighty, and despair!
        Nothing beside remains. Round the decay
        Of that colossal Wreck, boundless and bare
        The lone and level sands stretch far away
        ''',
 }



 @require_torch
 def      UpperCAmelCase							(					self							)  ->					Tuple:
        import torch

        _A        =  JukeboxTokenizer.from_pretrained("""openai/jukebox-1b-lyrics"""							)
        _A        =  tokenizer(**self.metas							)["""input_ids"""]
        # fmt: off
        _A        =  [
            torch.tensor([[
                0, 0, 0, 71_69, 5_07, 9, 76, 39, 31, 46, 76, 27,
                76, 46, 44, 27, 48, 31, 38, 38, 31, 44, 76, 32,
                44, 41, 39, 76, 27, 40, 76, 27, 40, 46, 35, 43,
                47, 31, 76, 38, 27, 40, 30, 64, 78, 76, 76, 76,
                76, 76, 76, 76, 76, 23, 34, 41, 76, 45, 27, 35,
                30, 76, 71, 20, 49, 41, 76, 48, 27, 45, 46, 76,
                27, 40, 30, 76, 46, 44, 47, 40, 37, 38, 31, 45,
                45, 76, 38, 31, 33, 45, 76, 41, 32, 76, 45, 46,
                41, 40, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                19, 46, 27, 40, 30, 76, 35, 40, 76, 46, 34, 31,
                76, 30, 31, 45, 31, 44, 46, 63, 76, 63, 76, 63,
                76, 63, 76, 14, 31, 27, 44, 76, 46, 34, 31, 39,
                64, 76, 41, 40, 76, 46, 34, 31, 76, 45, 27, 40,
                30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76, 8,
                27, 38, 32, 76, 45, 47, 40, 37, 76, 27, 76, 45,
                34, 27, 46, 46, 31, 44, 31, 30, 76, 48, 35, 45,
                27, 33, 31, 76, 38, 35, 31, 45, 64, 76, 49, 34,
                41, 45, 31, 76, 32, 44, 41, 49, 40, 64, 78, 76,
                76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76, 49,
                44, 35, 40, 37, 38, 31, 30, 76, 38, 35, 42, 64,
                76, 27, 40, 30, 76, 45, 40, 31, 31, 44, 76, 41,
                32, 76, 29, 41, 38, 30, 76, 29, 41, 39, 39, 27,
                40, 30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                20, 31, 38, 38, 76, 46, 34, 27, 46, 76, 35, 46,
                45, 76, 45, 29, 47, 38, 42, 46, 41, 44, 76, 49,
                31, 38, 38, 76, 46, 34, 41, 45, 31, 76, 42, 27,
                45, 45, 35, 41, 40, 45, 76, 44, 31, 27, 30, 78,
                76, 76, 76, 76, 76, 76, 76, 76, 23, 34, 35, 29,
                34, 76, 51, 31, 46, 76, 45, 47, 44, 48, 35, 48,
                31, 64, 76, 45, 46, 27, 39, 42, 31, 30, 76, 41,
                40, 76, 46, 34, 31, 45, 31, 76, 38, 35, 32, 31,
                38, 31, 45, 45, 76, 46, 34, 35, 40, 33, 45, 64,
                78, 76, 76, 76, 76, 76, 76, 76, 76, 20, 34, 31,
                76, 34, 27, 40, 30, 76, 46, 34, 27, 46, 76, 39,
                41, 29, 37, 31, 30, 76, 46, 34, 31, 39, 64, 76,
                27, 40, 30, 76, 46, 34, 31, 76, 34, 31, 27, 44,
                46, 76, 46, 34, 27, 46, 76, 32, 31, 30, 66, 78,
                76, 76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76,
                41, 40, 76, 46, 34, 31, 76, 42, 31, 30, 31, 45,
                46, 27, 38, 64, 76, 46, 34, 31, 45, 31, 76, 49,
                41, 44, 30, 45, 76, 27, 42, 42, 31, 27, 44, 65,
                78, 76, 76, 76, 76, 76, 76, 76, 76, 13, 51, 76,
                40, 27, 39, 31, 76, 35, 45, 76, 15, 52, 51, 39,
                27, 40, 30, 35, 27, 45, 64, 76, 11, 35, 40, 33,
                76, 41, 32, 76, 11, 35, 40, 33, 45, 66, 78, 76,
                76, 76, 76, 76, 76, 76, 76, 12, 41, 41, 37, 76,
                41, 40, 76, 39, 51, 76, 23, 41, 44, 37, 45, 64,
                76, 51, 31, 76, 13, 35, 33, 34, 46, 51, 64, 76,
                27, 40, 30, 76, 30, 31, 45, 42, 27, 35, 44, 67,
                78, 76, 76, 76, 76, 76, 76, 76, 76, 14, 41, 46,
                34, 35, 40, 33, 76, 28, 31, 45, 35, 30, 31, 76,
                44, 31, 39, 27, 35, 40, 45, 63, 76, 18, 41, 47,
                40, 30, 76, 46, 34, 31, 76, 30, 31, 29, 27, 51,
                78, 76, 76, 76, 76, 76, 76, 76, 76, 15, 32, 76,
                46, 34, 27, 46, 76, 29, 41, 38, 41, 45, 45, 27,
                38, 76, 23, 44, 31, 29, 37, 64, 76, 28, 41, 47,
                40, 30, 38, 31, 45, 45, 76, 27, 40, 30, 76, 28,
                27, 44, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                20, 34, 31, 76, 38, 41, 40, 31, 76, 27, 40, 30,
                76, 38, 31, 48, 31, 38, 76, 45, 27, 40, 30, 45,
                76, 45, 46, 44, 31, 46, 29, 34, 76, 32, 27, 44,
                76, 27, 49, 27, 51, 78, 76, 76, 76, 76, 76, 76,
                76, 76]]							),
            torch.tensor([[0, 0, 0, 10_69, 11]]							),
            torch.tensor([[0, 0, 0, 10_69, 11]]							),
        ]
        # fmt: on
        self.assertTrue(torch.allclose(tokens[0]		,				EXPECTED_OUTPUT[0]							)							)
        self.assertTrue(torch.allclose(tokens[1]		,				EXPECTED_OUTPUT[1]							)							)
        self.assertTrue(torch.allclose(tokens[2]		,				EXPECTED_OUTPUT[2]							)							)



 @require_torch
 def      UpperCAmelCase							(					self							)  ->					List[str]:
        import torch

        _A        =  JukeboxTokenizer.from_pretrained("""openai/jukebox-5b-lyrics"""							)
        _A        =  tokenizer(**self.metas							)["""input_ids"""]
        # fmt: off
        _A        =  [
            torch.tensor([[
                0, 0, 0, 10_69, 11, -1, -1, -1, -1, 9, 77, 39,
                31, 46, 77, 27, 77, 46, 44, 27, 48, 31, 38, 38,
                31, 44, 77, 32, 44, 41, 39, 77, 27, 40, 77, 27,
                40, 46, 35, 43, 47, 31, 77, 38, 27, 40, 30, 64,
                79, 77, 77, 77, 77, 77, 77, 77, 77, 23, 34, 41,
                77, 45, 27, 35, 30, 77, 72, 20, 49, 41, 77, 48,
                27, 45, 46, 77, 27, 40, 30, 77, 46, 44, 47, 40,
                37, 38, 31, 45, 45, 77, 38, 31, 33, 45, 77, 41,
                32, 77, 45, 46, 41, 40, 31, 79, 77, 77, 77, 77,
                77, 77, 77, 77, 19, 46, 27, 40, 30, 77, 35, 40,
                77, 46, 34, 31, 77, 30, 31, 45, 31, 44, 46, 63,
                77, 63, 77, 63, 77, 63, 77, 14, 31, 27, 44, 77,
                46, 34, 31, 39, 64, 77, 41, 40, 77, 46, 34, 31,
                77, 45, 27, 40, 30, 64, 79, 77, 77, 77, 77, 77,
                77, 77, 77, 8, 27, 38, 32, 77, 45, 47, 40, 37,
                77, 27, 77, 45, 34, 27, 46, 46, 31, 44, 31, 30,
                77, 48, 35, 45, 27, 33, 31, 77, 38, 35, 31, 45,
                64, 77, 49, 34, 41, 45, 31, 77, 32, 44, 41, 49,
                40, 64, 79, 77, 77, 77, 77, 77, 77, 77, 77, 1,
                40, 30, 77, 49, 44, 35, 40, 37, 38, 31, 30, 77,
                38, 35, 42, 64, 77, 27, 40, 30, 77, 45, 40, 31,
                31, 44, 77, 41, 32, 77, 29, 41, 38, 30, 77, 29,
                41, 39, 39, 27, 40, 30, 64, 79, 77, 77, 77, 77,
                77, 77, 77, 77, 20, 31, 38, 38, 77, 46, 34, 27,
                46, 77, 35, 46, 45, 77, 45, 29, 47, 38, 42, 46,
                41, 44, 77, 49, 31, 38, 38, 77, 46, 34, 41, 45,
                31, 77, 42, 27, 45, 45, 35, 41, 40, 45, 77, 44,
                31, 27, 30, 79, 77, 77, 77, 77, 77, 77, 77, 77,
                23, 34, 35, 29, 34, 77, 51, 31, 46, 77, 45, 47,
                44, 48, 35, 48, 31, 64, 77, 45, 46, 27, 39, 42,
                31, 30, 77, 41, 40, 77, 46, 34, 31, 45, 31, 77,
                38, 35, 32, 31, 38, 31, 45, 45, 77, 46, 34, 35,
                40, 33, 45, 64, 79, 77, 77, 77, 77, 77, 77, 77,
                77, 20, 34, 31, 77, 34, 27, 40, 30, 77, 46, 34,
                27, 46, 77, 39, 41, 29, 37, 31, 30, 77, 46, 34,
                31, 39, 64, 77, 27, 40, 30, 77, 46, 34, 31, 77,
                34, 31, 27, 44, 46, 77, 46, 34, 27, 46, 77, 32,
                31, 30, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77,
                1, 40, 30, 77, 41, 40, 77, 46, 34, 31, 77, 42,
                31, 30, 31, 45, 46, 27, 38, 64, 77, 46, 34, 31,
                45, 31, 77, 49, 41, 44, 30, 45, 77, 27, 42, 42,
                31, 27, 44, 65, 79, 77, 77, 77, 77, 77, 77, 77,
                77, 13, 51, 77, 40, 27, 39, 31, 77, 35, 45, 77,
                15, 52, 51, 39, 27, 40, 30, 35, 27, 45, 64, 77,
                11, 35, 40, 33, 77, 41, 32, 77, 11, 35, 40, 33,
                45, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77, 12,
                41, 41, 37, 77, 41, 40, 77, 39, 51, 77, 23, 41,
                44, 37, 45, 64, 77, 51, 31, 77, 13, 35, 33, 34,
                46, 51, 64, 77, 27, 40, 30, 77, 30, 31, 45, 42,
                27, 35, 44, 67, 79, 77, 77, 77, 77, 77, 77, 77,
                77, 14, 41, 46, 34, 35, 40, 33, 77, 28, 31, 45,
                35, 30, 31, 77, 44, 31, 39, 27, 35, 40, 45, 63,
                77, 18, 41, 47, 40, 30, 77, 46, 34, 31, 77, 30,
                31, 29, 27, 51, 79, 77, 77, 77, 77, 77, 77, 77,
                77, 15, 32, 77, 46, 34, 27, 46, 77, 29, 41, 38,
                41, 45, 45, 27, 38, 77, 23, 44, 31, 29, 37, 64,
                77, 28, 41, 47, 40, 30, 38, 31, 45, 45, 77, 27,
                40, 30, 77, 28, 27, 44, 31, 79, 77, 77, 77, 77,
                77, 77, 77, 77, 20, 34, 31, 77, 38, 41, 40, 31,
                77, 27, 40, 30, 77, 38, 31, 48, 31, 38, 77, 45,
                27, 40, 30, 45, 77, 45, 46, 44, 31, 46, 29, 34,
                77, 32, 27, 44, 77, 27, 49, 27, 51, 79, 77, 77,
                77, 77, 77, 77, 77, 77]]							),
            torch.tensor([[0, 0, 0, 10_69, 11, -1, -1, -1, -1]]							),
            torch.tensor([[0, 0, 0, 10_69, 11, -1, -1, -1, -1]]							),
        ]
        # fmt: on
        self.assertTrue(torch.allclose(tokens[0]		,				EXPECTED_OUTPUT[0]							)							)
        self.assertTrue(torch.allclose(tokens[1]		,				EXPECTED_OUTPUT[1]							)							)
        self.assertTrue(torch.allclose(tokens[2]		,				EXPECTED_OUTPUT[2]							)							)




