import json
import os
import shutil
import tempfile
import unittest
from multiprocessing import get_context
from pathlib import Path

import datasets
import numpy as np
from datasets import load_dataset
from parameterized import parameterized

from transformers import AutoProcessor
from transformers.models.wavaveca import WavaVecaCTCTokenizer, WavaVecaFeatureExtractor
from transformers.models.wavaveca.tokenization_wavaveca import VOCAB_FILES_NAMES
from transformers.testing_utils import require_pyctcdecode, require_torch, require_torchaudio, slow
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_pyctcdecode_available, is_torch_available

from ..wavaveca.test_feature_extraction_wavaveca import floats_list


if is_pyctcdecode_available():
    from huggingface_hub import snapshot_download
    from pyctcdecode import BeamSearchDecoderCTC

    from transformers.models.wavaveca_with_lm import WavaVecaProcessorWithLM
    from transformers.models.wavaveca_with_lm.processing_wavaveca_with_lm import WavaVecaDecoderWithLMOutput

if is_torch_available():
    from transformers import WavaVecaForCTC



@require_pyctcdecode
class 				a							(			unittest.TestCase       ):







 """simple docstring"""






 def      UpperCAmelCase							(					self							)  ->					Union[str, Any]:
        _A        =  """| <pad> <unk> <s> </s> a b c d e f g h i j k""".split()
        _A        =  dict(zip(lowerCAmelCase_		,				range(len(lowerCAmelCase_							)							)							)							)

        _A        =  {
            """unk_token""": """<unk>""",
            """bos_token""": """<s>""",
            """eos_token""": """</s>""",
        }
        _A        =  {
            """feature_size""": 1,
            """padding_value""": 0.0,
            """sampling_rate""": 1_60_00,
            """return_attention_mask""": False,
            """do_normalize""": True,
        }

        _A        =  tempfile.mkdtemp()
        _A        =  os.path.join(self.tmpdirname		,				VOCAB_FILES_NAMES["""vocab_file"""]							)
        _A        =  os.path.join(self.tmpdirname		,				lowerCAmelCase_							)
        with open(self.vocab_file		,				"""w"""		,				encoding="""utf-8"""							) as fp:
               fp.write(json.dumps(lowerCAmelCase_							) + """\n"""							)

        with open(self.feature_extraction_file		,				"""w"""		,				encoding="""utf-8"""							) as fp:
               fp.write(json.dumps(lowerCAmelCase_							) + """\n"""							)

        # load decoder from hub
        _A        =  """hf-internal-testing/ngram-beam-search-decoder"""



 def      UpperCAmelCase							(					self		,				**lowerCAmelCase_							)  ->					Tuple:
        _A        =  self.add_kwargs_tokens_map.copy()
        kwargs.update(lowerCAmelCase_							)
        return WavaVecaCTCTokenizer.from_pretrained(self.tmpdirname		,				**lowerCAmelCase_							)



 def      UpperCAmelCase							(					self		,				**lowerCAmelCase_							)  ->					Any:
        return WavaVecaFeatureExtractor.from_pretrained(self.tmpdirname		,				**lowerCAmelCase_							)



 def      UpperCAmelCase							(					self		,				**lowerCAmelCase_							)  ->					Any:
        return BeamSearchDecoderCTC.load_from_hf_hub(self.decoder_name		,				**lowerCAmelCase_							)



 def      UpperCAmelCase							(					self							)  ->					List[str]:
        shutil.rmtree(self.tmpdirname							)



 def      UpperCAmelCase							(					self							)  ->					Tuple:
        _A        =  self.get_tokenizer()
        _A        =  self.get_feature_extractor()
        _A        =  self.get_decoder()

        _A        =  WavaVecaProcessorWithLM(tokenizer=lowerCAmelCase_		,				feature_extractor=lowerCAmelCase_		,				decoder=lowerCAmelCase_							)

        processor.save_pretrained(self.tmpdirname							)
        _A        =  WavaVecaProcessorWithLM.from_pretrained(self.tmpdirname							)

        # tokenizer
        self.assertEqual(processor.tokenizer.get_vocab()		,				tokenizer.get_vocab()							)
        self.assertIsInstance(processor.tokenizer		,				lowerCAmelCase_							)

        # feature extractor
        self.assertEqual(processor.feature_extractor.to_json_string()		,				feature_extractor.to_json_string()							)
        self.assertIsInstance(processor.feature_extractor		,				lowerCAmelCase_							)

        # decoder
        self.assertEqual(processor.decoder._alphabet.labels		,				decoder._alphabet.labels							)
        self.assertEqual(
            processor.decoder.model_container[decoder._model_key]._unigram_set		,				decoder.model_container[decoder._model_key]._unigram_set		,				)
        self.assertIsInstance(processor.decoder		,				lowerCAmelCase_							)



 def      UpperCAmelCase							(					self							)  ->					Tuple:
        _A        =  WavaVecaProcessorWithLM(
            tokenizer=self.get_tokenizer()		,				feature_extractor=self.get_feature_extractor()		,				decoder=self.get_decoder()							)
        processor.save_pretrained(self.tmpdirname							)

        # make sure that error is thrown when decoder alphabet doesn't match
        _A        =  WavaVecaProcessorWithLM.from_pretrained(
            self.tmpdirname		,				alpha=5.0		,				beta=3.0		,				score_boundary=-7.0		,				unk_score_offset=3							)

        # decoder
        self.assertEqual(processor.language_model.alpha		,				5.0							)
        self.assertEqual(processor.language_model.beta		,				3.0							)
        self.assertEqual(processor.language_model.score_boundary		,				-7.0							)
        self.assertEqual(processor.language_model.unk_score_offset		,				3							)



 def      UpperCAmelCase							(					self							)  ->					Optional[int]:
        _A        =  self.get_tokenizer()
        # add token to trigger raise
        tokenizer.add_tokens(["""xx"""]							)
        with self.assertRaisesRegex(lowerCAmelCase_		,				"""include"""							):
               WavaVecaProcessorWithLM(
                   tokenizer=lowerCAmelCase_		,				feature_extractor=self.get_feature_extractor()		,				decoder=self.get_decoder()							)



 def      UpperCAmelCase							(					self							)  ->					Any:
        _A        =  self.get_feature_extractor()
        _A        =  self.get_tokenizer()
        _A        =  self.get_decoder()

        _A        =  WavaVecaProcessorWithLM(tokenizer=lowerCAmelCase_		,				feature_extractor=lowerCAmelCase_		,				decoder=lowerCAmelCase_							)

        _A        =  floats_list((3, 10_00)							)

        _A        =  feature_extractor(lowerCAmelCase_		,				return_tensors="""np"""							)
        _A        =  processor(lowerCAmelCase_		,				return_tensors="""np"""							)

        for key in input_feat_extract.keys():
               self.assertAlmostEqual(input_feat_extract[key].sum()		,				input_processor[key].sum()		,				delta=1E-2							)



 def      UpperCAmelCase							(					self							)  ->					Any:
        _A        =  self.get_feature_extractor()
        _A        =  self.get_tokenizer()
        _A        =  self.get_decoder()

        _A        =  WavaVecaProcessorWithLM(tokenizer=lowerCAmelCase_		,				feature_extractor=lowerCAmelCase_		,				decoder=lowerCAmelCase_							)

        _A        =  """This is a test string"""

        _A        =  processor(text=lowerCAmelCase_							)

        _A        =  tokenizer(lowerCAmelCase_							)

        for key in encoded_tok.keys():
               self.assertListEqual(encoded_tok[key]		,				encoded_processor[key]							)



 def      UpperCAmelCase							(					self		,				lowerCAmelCase_=(2, 10, 16)		,				lowerCAmelCase_=77							)  ->					Tuple:
        np.random.seed(lowerCAmelCase_							)
        return np.random.rand(*lowerCAmelCase_							)



 def      UpperCAmelCase							(					self							)  ->					Tuple:
        _A        =  self.get_feature_extractor()
        _A        =  self.get_tokenizer()
        _A        =  self.get_decoder()

        _A        =  WavaVecaProcessorWithLM(tokenizer=lowerCAmelCase_		,				feature_extractor=lowerCAmelCase_		,				decoder=lowerCAmelCase_							)

        _A        =  self._get_dummy_logits(shape=(10, 16)		,				seed=13							)

        _A        =  processor.decode(lowerCAmelCase_							)

        _A        =  decoder.decode_beams(lowerCAmelCase_							)[0]

        self.assertEqual(decoded_decoder[0]		,				decoded_processor.text							)
        self.assertEqual("""</s> <s> </s>"""		,				decoded_processor.text							)
        self.assertEqual(decoded_decoder[-2]		,				decoded_processor.logit_score							)
        self.assertEqual(decoded_decoder[-1]		,				decoded_processor.lm_score							)



 @parameterized.expand([[None], ["""fork"""], ["""spawn"""]]							)
 def      UpperCAmelCase							(					self		,				lowerCAmelCase_							)  ->					Optional[int]:
        _A        =  self.get_feature_extractor()
        _A        =  self.get_tokenizer()
        _A        =  self.get_decoder()

        _A        =  WavaVecaProcessorWithLM(tokenizer=lowerCAmelCase_		,				feature_extractor=lowerCAmelCase_		,				decoder=lowerCAmelCase_							)

        _A        =  self._get_dummy_logits()

        # note: pool should be instantiated *after* Wav2Vec2ProcessorWithLM.
        #       otherwise, the LM won't be available to the pool's sub-processes.
        # manual logic used to allow parameterized test for both pool=None and pool=Pool(...)
        if pool_context is None:
               _A        =  processor.batch_decode(lowerCAmelCase_							)
        else:
               with get_context(lowerCAmelCase_							).Pool() as pool:
                      _A        =  processor.batch_decode(lowerCAmelCase_		,				lowerCAmelCase_							)

        _A        =  list(lowerCAmelCase_							)

        with get_context("""fork"""							).Pool() as p:
               _A        =  decoder.decode_beams_batch(lowerCAmelCase_		,				lowerCAmelCase_							)

        _A   ,  _A   ,  _A        =  [], [], []
        for beams in decoded_beams:
               texts_decoder.append(beams[0][0]							)
               logit_scores_decoder.append(beams[0][-2]							)
               lm_scores_decoder.append(beams[0][-1]							)

        self.assertListEqual(lowerCAmelCase_		,				decoded_processor.text							)
        self.assertListEqual(["""<s> <s> </s>""", """<s> <s> <s>"""]		,				decoded_processor.text							)
        self.assertListEqual(lowerCAmelCase_		,				decoded_processor.logit_score							)
        self.assertListEqual(lowerCAmelCase_		,				decoded_processor.lm_score							)



 def      UpperCAmelCase							(					self							)  ->					List[Any]:
        _A        =  self.get_feature_extractor()
        _A        =  self.get_tokenizer()
        _A        =  self.get_decoder()

        _A        =  WavaVecaProcessorWithLM(tokenizer=lowerCAmelCase_		,				feature_extractor=lowerCAmelCase_		,				decoder=lowerCAmelCase_							)

        _A        =  self._get_dummy_logits()

        _A        =  15
        _A        =  -20.0
        _A        =  -4.0

        _A        =  processor.batch_decode(
            lowerCAmelCase_		,				beam_width=lowerCAmelCase_		,				beam_prune_logp=lowerCAmelCase_		,				token_min_logp=lowerCAmelCase_		,				)
        _A        =  decoded_processor_out.text

        _A        =  list(lowerCAmelCase_							)

        with get_context("""fork"""							).Pool() as pool:
               _A        =  decoder.decode_beams_batch(
                   lowerCAmelCase_		,				lowerCAmelCase_		,				beam_width=lowerCAmelCase_		,				beam_prune_logp=lowerCAmelCase_		,				token_min_logp=lowerCAmelCase_		,				)

        _A        =  [d[0][0] for d in decoded_decoder_out]
        _A        =  [d[0][2] for d in decoded_decoder_out]
        _A        =  [d[0][3] for d in decoded_decoder_out]

        self.assertListEqual(lowerCAmelCase_		,				lowerCAmelCase_							)
        self.assertListEqual(["""</s> <s> <s>""", """<s> <s> <s>"""]		,				lowerCAmelCase_							)

        self.assertTrue(np.array_equal(lowerCAmelCase_		,				decoded_processor_out.logit_score							)							)
        self.assertTrue(np.allclose([-20.054, -18.447]		,				lowerCAmelCase_		,				atol=1E-3							)							)

        self.assertTrue(np.array_equal(lowerCAmelCase_		,				decoded_processor_out.lm_score							)							)
        self.assertTrue(np.allclose([-15.554, -13.9474]		,				lowerCAmelCase_		,				atol=1E-3							)							)



 def      UpperCAmelCase							(					self							)  ->					List[Any]:
        _A        =  self.get_feature_extractor()
        _A        =  self.get_tokenizer()
        _A        =  self.get_decoder()

        _A        =  WavaVecaProcessorWithLM(tokenizer=lowerCAmelCase_		,				feature_extractor=lowerCAmelCase_		,				decoder=lowerCAmelCase_							)

        _A        =  self._get_dummy_logits()

        _A        =  2.0
        _A        =  5.0
        _A        =  -20.0
        _A        =  True

        _A        =  processor.batch_decode(
            lowerCAmelCase_		,				alpha=lowerCAmelCase_		,				beta=lowerCAmelCase_		,				unk_score_offset=lowerCAmelCase_		,				lm_score_boundary=lowerCAmelCase_		,				)
        _A        =  decoded_processor_out.text

        _A        =  list(lowerCAmelCase_							)
        decoder.reset_params(
            alpha=lowerCAmelCase_		,				beta=lowerCAmelCase_		,				unk_score_offset=lowerCAmelCase_		,				lm_score_boundary=lowerCAmelCase_		,				)

        with get_context("""fork"""							).Pool() as pool:
               _A        =  decoder.decode_beams_batch(
                   lowerCAmelCase_		,				lowerCAmelCase_		,				)

        _A        =  [d[0][0] for d in decoded_decoder_out]

        self.assertListEqual(lowerCAmelCase_		,				lowerCAmelCase_							)
        self.assertListEqual(["""<s> </s> <s> </s> </s>""", """</s> </s> <s> </s> </s>"""]		,				lowerCAmelCase_							)
        _A        =  processor.decoder.model_container[processor.decoder._model_key]
        self.assertEqual(lm_model.alpha		,				2.0							)
        self.assertEqual(lm_model.beta		,				5.0							)
        self.assertEqual(lm_model.unk_score_offset		,				-20.0							)
        self.assertEqual(lm_model.score_boundary		,				lowerCAmelCase_							)



 def      UpperCAmelCase							(					self							)  ->					Union[str, Any]:
        _A        =  WavaVecaProcessorWithLM.from_pretrained("""hf-internal-testing/processor_with_lm"""							)

        _A        =  processor.decoder.model_container[processor.decoder._model_key]
        _A        =  Path(language_model._kenlm_model.path.decode("""utf-8"""							)							).parent.parent.absolute()

        _A        =  os.listdir(lowerCAmelCase_							)
        _A        =  ["""alphabet.json""", """language_model"""]

        downloaded_decoder_files.sort()
        expected_decoder_files.sort()

        # test that only decoder relevant files from
        # https://huggingface.co/hf-internal-testing/processor_with_lm/tree/main
        # are downloaded and none of the rest (e.g. README.md, ...)
        self.assertListEqual(lowerCAmelCase_		,				lowerCAmelCase_							)



 def      UpperCAmelCase							(					self							)  ->					List[str]:
        _A        =  snapshot_download("""hf-internal-testing/processor_with_lm"""							)

        _A        =  WavaVecaProcessorWithLM.from_pretrained(lowerCAmelCase_							)

        _A        =  processor.decoder.model_container[processor.decoder._model_key]
        _A        =  Path(language_model._kenlm_model.path.decode("""utf-8"""							)							).parent.parent.absolute()

        _A        =  os.listdir(lowerCAmelCase_							)
        _A        =  os.listdir(lowerCAmelCase_							)

        local_decoder_files.sort()
        expected_decoder_files.sort()

        # test that both decoder form hub and local files in cache are the same
        self.assertListEqual(lowerCAmelCase_		,				lowerCAmelCase_							)



 def      UpperCAmelCase							(					self							)  ->					List[str]:
        _A        =  WavaVecaProcessorWithLM.from_pretrained("""hf-internal-testing/processor_with_lm"""							)
        _A        =  AutoProcessor.from_pretrained("""hf-internal-testing/processor_with_lm"""							)

        _A        =  floats_list((3, 10_00)							)

        _A        =  processor_wavaveca(lowerCAmelCase_		,				return_tensors="""np"""							)
        _A        =  processor_auto(lowerCAmelCase_		,				return_tensors="""np"""							)

        for key in input_wavaveca.keys():
               self.assertAlmostEqual(input_wavaveca[key].sum()		,				input_auto[key].sum()		,				delta=1E-2							)

        _A        =  self._get_dummy_logits()

        _A        =  processor_wavaveca.batch_decode(lowerCAmelCase_							)
        _A        =  processor_auto.batch_decode(lowerCAmelCase_							)

        self.assertListEqual(decoded_wavaveca.text		,				decoded_auto.text							)



 def      UpperCAmelCase							(					self							)  ->					Union[str, Any]:
        _A        =  self.get_feature_extractor()
        _A        =  self.get_tokenizer()
        _A        =  self.get_decoder()

        _A        =  WavaVecaProcessorWithLM(tokenizer=lowerCAmelCase_		,				feature_extractor=lowerCAmelCase_		,				decoder=lowerCAmelCase_							)

        self.assertListEqual(
            processor.model_input_names		,				feature_extractor.model_input_names		,				msg="""`processor` and `feature_extractor` model input names do not match"""		,				)



 @staticmethod
 def      UpperCAmelCase							(					lowerCAmelCase_		,				lowerCAmelCase_							)  ->					int:
        _A        =  [d[key] for d in offsets]
        return retrieved_list



 def      UpperCAmelCase							(					self							)  ->					Any:
        _A        =  WavaVecaProcessorWithLM.from_pretrained("""hf-internal-testing/processor_with_lm"""							)
        _A        =  self._get_dummy_logits()[0]

        _A        =  processor.decode(lowerCAmelCase_		,				output_word_offsets=lowerCAmelCase_							)
        # check Wav2Vec2CTCTokenizerOutput keys for word
        self.assertEqual(len(outputs.keys()							)		,				4							)
        self.assertTrue("""text""" in outputs							)
        self.assertTrue("""word_offsets""" in outputs							)
        self.assertTrue(isinstance(lowerCAmelCase_		,				lowerCAmelCase_							)							)

        self.assertEqual(""" """.join(self.get_from_offsets(outputs["""word_offsets"""]		,				"""word"""							)							)		,				outputs.text							)
        self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""]		,				"""word"""							)		,				["""<s>""", """<s>""", """</s>"""]							)
        self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""]		,				"""start_offset"""							)		,				[0, 2, 4]							)
        self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""]		,				"""end_offset"""							)		,				[1, 3, 5]							)



 def      UpperCAmelCase							(					self							)  ->					Any:
        _A        =  WavaVecaProcessorWithLM.from_pretrained("""hf-internal-testing/processor_with_lm"""							)
        _A        =  self._get_dummy_logits()

        _A        =  processor.batch_decode(lowerCAmelCase_		,				output_word_offsets=lowerCAmelCase_							)

        # check Wav2Vec2CTCTokenizerOutput keys for word
        self.assertEqual(len(outputs.keys()							)		,				4							)
        self.assertTrue("""text""" in outputs							)
        self.assertTrue("""word_offsets""" in outputs							)
        self.assertTrue(isinstance(lowerCAmelCase_		,				lowerCAmelCase_							)							)

        self.assertListEqual(
            [""" """.join(self.get_from_offsets(lowerCAmelCase_		,				"""word"""							)							) for o in outputs["""word_offsets"""]]		,				outputs.text							)
        self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""][0]		,				"""word"""							)		,				["""<s>""", """<s>""", """</s>"""]							)
        self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""][0]		,				"""start_offset"""							)		,				[0, 2, 4]							)
        self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""][0]		,				"""end_offset"""							)		,				[1, 3, 5]							)



 @slow
 @require_torch
 @require_torchaudio
 def      UpperCAmelCase							(					self							)  ->					Any:
        import torch

        _A        =  load_dataset("""common_voice"""		,				"""en"""		,				split="""train"""		,				streaming=lowerCAmelCase_							)
        _A        =  ds.cast_column("""audio"""		,				datasets.Audio(sampling_rate=1_60_00							)							)
        _A        =  iter(lowerCAmelCase_							)
        _A        =  next(lowerCAmelCase_							)

        _A        =  AutoProcessor.from_pretrained("""patrickvonplaten/wav2vec2-base-100h-with-lm"""							)
        _A        =  WavaVecaForCTC.from_pretrained("""patrickvonplaten/wav2vec2-base-100h-with-lm"""							)

        # compare to filename `common_voice_en_100038.mp3` of dataset viewer on https://huggingface.co/datasets/common_voice/viewer/en/train
        _A        =  processor(sample["""audio"""]["""array"""]		,				return_tensors="""pt"""							).input_values

        with torch.no_grad():
               _A        =  model(lowerCAmelCase_							).logits.cpu().numpy()

        _A        =  processor.decode(logits[0]		,				output_word_offsets=lowerCAmelCase_							)

        _A        =  model.config.inputs_to_logits_ratio / processor.feature_extractor.sampling_rate
        _A        =  [
            {
                """start_time""": d["""start_offset"""] * time_offset,
                """end_time""": d["""end_offset"""] * time_offset,
                """word""": d["""word"""],
            }
            for d in output["""word_offsets"""]
        ]

        _A        =  """WHY DOES MILISANDRA LOOK LIKE SHE WANTS TO CONSUME JOHN SNOW ON THE RIVER AT THE WALL"""

        # output words
        self.assertEqual(""" """.join(self.get_from_offsets(lowerCAmelCase_		,				"""word"""							)							)		,				lowerCAmelCase_							)
        self.assertEqual(""" """.join(self.get_from_offsets(lowerCAmelCase_		,				"""word"""							)							)		,				output.text							)

        # output times
        _A        =  torch.tensor(self.get_from_offsets(lowerCAmelCase_		,				"""start_time"""							)							)
        _A        =  torch.tensor(self.get_from_offsets(lowerCAmelCase_		,				"""end_time"""							)							)

        # fmt: off
        _A        =  torch.tensor([1.4199, 1.6599, 2.2599, 3.0, 3.24, 3.5999, 3.7999, 4.0999, 4.26, 4.94, 5.28, 5.6599, 5.78, 5.94, 6.32, 6.5399, 6.6599]							)
        _A        =  torch.tensor([1.5399, 1.8999, 2.9, 3.16, 3.5399, 3.72, 4.0199, 4.1799, 4.76, 5.1599, 5.5599, 5.6999, 5.86, 6.1999, 6.38, 6.6199, 6.94]							)
        # fmt: on

        self.assertTrue(torch.allclose(lowerCAmelCase_		,				lowerCAmelCase_		,				atol=0.01							)							)
        self.assertTrue(torch.allclose(lowerCAmelCase_		,				lowerCAmelCase_		,				atol=0.01							)							)





import unittest

from transformers import JukeboxTokenizer
from transformers.testing_utils import require_torch



class 				a							(			unittest.TestCase       ):







 """simple docstring"""






 lowerCamelCase	:Tuple    			=		JukeboxTokenizer
 lowerCamelCase	:str    			=		{
     '''artist''': '''Zac Brown Band''',
     '''genres''': '''Country''',
     '''lyrics''': '''I met a traveller from an antique land,
        Who said "Two vast and trunkless legs of stone
        Stand in the desert. . . . Near them, on the sand,
        Half sunk a shattered visage lies, whose frown,
        And wrinkled lip, and sneer of cold command,
        Tell that its sculptor well those passions read
        Which yet survive, stamped on these lifeless things,
        The hand that mocked them, and the heart that fed;
        And on the pedestal, these words appear:
        My name is Ozymandias, King of Kings;
        Look on my Works, ye Mighty, and despair!
        Nothing beside remains. Round the decay
        Of that colossal Wreck, boundless and bare
        The lone and level sands stretch far away
        ''',
 }



 @require_torch
 def      UpperCAmelCase							(					self							)  ->					Tuple:
        import torch

        _A        =  JukeboxTokenizer.from_pretrained("""openai/jukebox-1b-lyrics"""							)
        _A        =  tokenizer(**self.metas							)["""input_ids"""]
        # fmt: off
        _A        =  [
            torch.tensor([[
                0, 0, 0, 71_69, 5_07, 9, 76, 39, 31, 46, 76, 27,
                76, 46, 44, 27, 48, 31, 38, 38, 31, 44, 76, 32,
                44, 41, 39, 76, 27, 40, 76, 27, 40, 46, 35, 43,
                47, 31, 76, 38, 27, 40, 30, 64, 78, 76, 76, 76,
                76, 76, 76, 76, 76, 23, 34, 41, 76, 45, 27, 35,
                30, 76, 71, 20, 49, 41, 76, 48, 27, 45, 46, 76,
                27, 40, 30, 76, 46, 44, 47, 40, 37, 38, 31, 45,
                45, 76, 38, 31, 33, 45, 76, 41, 32, 76, 45, 46,
                41, 40, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                19, 46, 27, 40, 30, 76, 35, 40, 76, 46, 34, 31,
                76, 30, 31, 45, 31, 44, 46, 63, 76, 63, 76, 63,
                76, 63, 76, 14, 31, 27, 44, 76, 46, 34, 31, 39,
                64, 76, 41, 40, 76, 46, 34, 31, 76, 45, 27, 40,
                30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76, 8,
                27, 38, 32, 76, 45, 47, 40, 37, 76, 27, 76, 45,
                34, 27, 46, 46, 31, 44, 31, 30, 76, 48, 35, 45,
                27, 33, 31, 76, 38, 35, 31, 45, 64, 76, 49, 34,
                41, 45, 31, 76, 32, 44, 41, 49, 40, 64, 78, 76,
                76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76, 49,
                44, 35, 40, 37, 38, 31, 30, 76, 38, 35, 42, 64,
                76, 27, 40, 30, 76, 45, 40, 31, 31, 44, 76, 41,
                32, 76, 29, 41, 38, 30, 76, 29, 41, 39, 39, 27,
                40, 30, 64, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                20, 31, 38, 38, 76, 46, 34, 27, 46, 76, 35, 46,
                45, 76, 45, 29, 47, 38, 42, 46, 41, 44, 76, 49,
                31, 38, 38, 76, 46, 34, 41, 45, 31, 76, 42, 27,
                45, 45, 35, 41, 40, 45, 76, 44, 31, 27, 30, 78,
                76, 76, 76, 76, 76, 76, 76, 76, 23, 34, 35, 29,
                34, 76, 51, 31, 46, 76, 45, 47, 44, 48, 35, 48,
                31, 64, 76, 45, 46, 27, 39, 42, 31, 30, 76, 41,
                40, 76, 46, 34, 31, 45, 31, 76, 38, 35, 32, 31,
                38, 31, 45, 45, 76, 46, 34, 35, 40, 33, 45, 64,
                78, 76, 76, 76, 76, 76, 76, 76, 76, 20, 34, 31,
                76, 34, 27, 40, 30, 76, 46, 34, 27, 46, 76, 39,
                41, 29, 37, 31, 30, 76, 46, 34, 31, 39, 64, 76,
                27, 40, 30, 76, 46, 34, 31, 76, 34, 31, 27, 44,
                46, 76, 46, 34, 27, 46, 76, 32, 31, 30, 66, 78,
                76, 76, 76, 76, 76, 76, 76, 76, 1, 40, 30, 76,
                41, 40, 76, 46, 34, 31, 76, 42, 31, 30, 31, 45,
                46, 27, 38, 64, 76, 46, 34, 31, 45, 31, 76, 49,
                41, 44, 30, 45, 76, 27, 42, 42, 31, 27, 44, 65,
                78, 76, 76, 76, 76, 76, 76, 76, 76, 13, 51, 76,
                40, 27, 39, 31, 76, 35, 45, 76, 15, 52, 51, 39,
                27, 40, 30, 35, 27, 45, 64, 76, 11, 35, 40, 33,
                76, 41, 32, 76, 11, 35, 40, 33, 45, 66, 78, 76,
                76, 76, 76, 76, 76, 76, 76, 12, 41, 41, 37, 76,
                41, 40, 76, 39, 51, 76, 23, 41, 44, 37, 45, 64,
                76, 51, 31, 76, 13, 35, 33, 34, 46, 51, 64, 76,
                27, 40, 30, 76, 30, 31, 45, 42, 27, 35, 44, 67,
                78, 76, 76, 76, 76, 76, 76, 76, 76, 14, 41, 46,
                34, 35, 40, 33, 76, 28, 31, 45, 35, 30, 31, 76,
                44, 31, 39, 27, 35, 40, 45, 63, 76, 18, 41, 47,
                40, 30, 76, 46, 34, 31, 76, 30, 31, 29, 27, 51,
                78, 76, 76, 76, 76, 76, 76, 76, 76, 15, 32, 76,
                46, 34, 27, 46, 76, 29, 41, 38, 41, 45, 45, 27,
                38, 76, 23, 44, 31, 29, 37, 64, 76, 28, 41, 47,
                40, 30, 38, 31, 45, 45, 76, 27, 40, 30, 76, 28,
                27, 44, 31, 78, 76, 76, 76, 76, 76, 76, 76, 76,
                20, 34, 31, 76, 38, 41, 40, 31, 76, 27, 40, 30,
                76, 38, 31, 48, 31, 38, 76, 45, 27, 40, 30, 45,
                76, 45, 46, 44, 31, 46, 29, 34, 76, 32, 27, 44,
                76, 27, 49, 27, 51, 78, 76, 76, 76, 76, 76, 76,
                76, 76]]							),
            torch.tensor([[0, 0, 0, 10_69, 11]]							),
            torch.tensor([[0, 0, 0, 10_69, 11]]							),
        ]
        # fmt: on
        self.assertTrue(torch.allclose(tokens[0]		,				EXPECTED_OUTPUT[0]							)							)
        self.assertTrue(torch.allclose(tokens[1]		,				EXPECTED_OUTPUT[1]							)							)
        self.assertTrue(torch.allclose(tokens[2]		,				EXPECTED_OUTPUT[2]							)							)



 @require_torch
 def      UpperCAmelCase							(					self							)  ->					List[str]:
        import torch

        _A        =  JukeboxTokenizer.from_pretrained("""openai/jukebox-5b-lyrics"""							)
        _A        =  tokenizer(**self.metas							)["""input_ids"""]
        # fmt: off
        _A        =  [
            torch.tensor([[
                0, 0, 0, 10_69, 11, -1, -1, -1, -1, 9, 77, 39,
                31, 46, 77, 27, 77, 46, 44, 27, 48, 31, 38, 38,
                31, 44, 77, 32, 44, 41, 39, 77, 27, 40, 77, 27,
                40, 46, 35, 43, 47, 31, 77, 38, 27, 40, 30, 64,
                79, 77, 77, 77, 77, 77, 77, 77, 77, 23, 34, 41,
                77, 45, 27, 35, 30, 77, 72, 20, 49, 41, 77, 48,
                27, 45, 46, 77, 27, 40, 30, 77, 46, 44, 47, 40,
                37, 38, 31, 45, 45, 77, 38, 31, 33, 45, 77, 41,
                32, 77, 45, 46, 41, 40, 31, 79, 77, 77, 77, 77,
                77, 77, 77, 77, 19, 46, 27, 40, 30, 77, 35, 40,
                77, 46, 34, 31, 77, 30, 31, 45, 31, 44, 46, 63,
                77, 63, 77, 63, 77, 63, 77, 14, 31, 27, 44, 77,
                46, 34, 31, 39, 64, 77, 41, 40, 77, 46, 34, 31,
                77, 45, 27, 40, 30, 64, 79, 77, 77, 77, 77, 77,
                77, 77, 77, 8, 27, 38, 32, 77, 45, 47, 40, 37,
                77, 27, 77, 45, 34, 27, 46, 46, 31, 44, 31, 30,
                77, 48, 35, 45, 27, 33, 31, 77, 38, 35, 31, 45,
                64, 77, 49, 34, 41, 45, 31, 77, 32, 44, 41, 49,
                40, 64, 79, 77, 77, 77, 77, 77, 77, 77, 77, 1,
                40, 30, 77, 49, 44, 35, 40, 37, 38, 31, 30, 77,
                38, 35, 42, 64, 77, 27, 40, 30, 77, 45, 40, 31,
                31, 44, 77, 41, 32, 77, 29, 41, 38, 30, 77, 29,
                41, 39, 39, 27, 40, 30, 64, 79, 77, 77, 77, 77,
                77, 77, 77, 77, 20, 31, 38, 38, 77, 46, 34, 27,
                46, 77, 35, 46, 45, 77, 45, 29, 47, 38, 42, 46,
                41, 44, 77, 49, 31, 38, 38, 77, 46, 34, 41, 45,
                31, 77, 42, 27, 45, 45, 35, 41, 40, 45, 77, 44,
                31, 27, 30, 79, 77, 77, 77, 77, 77, 77, 77, 77,
                23, 34, 35, 29, 34, 77, 51, 31, 46, 77, 45, 47,
                44, 48, 35, 48, 31, 64, 77, 45, 46, 27, 39, 42,
                31, 30, 77, 41, 40, 77, 46, 34, 31, 45, 31, 77,
                38, 35, 32, 31, 38, 31, 45, 45, 77, 46, 34, 35,
                40, 33, 45, 64, 79, 77, 77, 77, 77, 77, 77, 77,
                77, 20, 34, 31, 77, 34, 27, 40, 30, 77, 46, 34,
                27, 46, 77, 39, 41, 29, 37, 31, 30, 77, 46, 34,
                31, 39, 64, 77, 27, 40, 30, 77, 46, 34, 31, 77,
                34, 31, 27, 44, 46, 77, 46, 34, 27, 46, 77, 32,
                31, 30, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77,
                1, 40, 30, 77, 41, 40, 77, 46, 34, 31, 77, 42,
                31, 30, 31, 45, 46, 27, 38, 64, 77, 46, 34, 31,
                45, 31, 77, 49, 41, 44, 30, 45, 77, 27, 42, 42,
                31, 27, 44, 65, 79, 77, 77, 77, 77, 77, 77, 77,
                77, 13, 51, 77, 40, 27, 39, 31, 77, 35, 45, 77,
                15, 52, 51, 39, 27, 40, 30, 35, 27, 45, 64, 77,
                11, 35, 40, 33, 77, 41, 32, 77, 11, 35, 40, 33,
                45, 66, 79, 77, 77, 77, 77, 77, 77, 77, 77, 12,
                41, 41, 37, 77, 41, 40, 77, 39, 51, 77, 23, 41,
                44, 37, 45, 64, 77, 51, 31, 77, 13, 35, 33, 34,
                46, 51, 64, 77, 27, 40, 30, 77, 30, 31, 45, 42,
                27, 35, 44, 67, 79, 77, 77, 77, 77, 77, 77, 77,
                77, 14, 41, 46, 34, 35, 40, 33, 77, 28, 31, 45,
                35, 30, 31, 77, 44, 31, 39, 27, 35, 40, 45, 63,
                77, 18, 41, 47, 40, 30, 77, 46, 34, 31, 77, 30,
                31, 29, 27, 51, 79, 77, 77, 77, 77, 77, 77, 77,
                77, 15, 32, 77, 46, 34, 27, 46, 77, 29, 41, 38,
                41, 45, 45, 27, 38, 77, 23, 44, 31, 29, 37, 64,
                77, 28, 41, 47, 40, 30, 38, 31, 45, 45, 77, 27,
                40, 30, 77, 28, 27, 44, 31, 79, 77, 77, 77, 77,
                77, 77, 77, 77, 20, 34, 31, 77, 38, 41, 40, 31,
                77, 27, 40, 30, 77, 38, 31, 48, 31, 38, 77, 45,
                27, 40, 30, 45, 77, 45, 46, 44, 31, 46, 29, 34,
                77, 32, 27, 44, 77, 27, 49, 27, 51, 79, 77, 77,
                77, 77, 77, 77, 77, 77]]							),
            torch.tensor([[0, 0, 0, 10_69, 11, -1, -1, -1, -1]]							),
            torch.tensor([[0, 0, 0, 10_69, 11, -1, -1, -1, -1]]							),
        ]
        # fmt: on
        self.assertTrue(torch.allclose(tokens[0]		,				EXPECTED_OUTPUT[0]							)							)
        self.assertTrue(torch.allclose(tokens[1]		,				EXPECTED_OUTPUT[1]							)							)
        self.assertTrue(torch.allclose(tokens[2]		,				EXPECTED_OUTPUT[2]							)							)




