code: 
  
  import random
  import shutil
  import tempfile
  import unittest
  
  import numpy as np
  
  from transformers import TaTokenizer, TaTokenizerFast
  from transformers.testing_utils import require_sentencepiece, require_torch
  from transformers.utils.import_utils import is_speech_available, is_torch_available
  
  
  if is_torch_available():
  			pass
  
  if is_speech_available():
  			from transformers import EncodecFeatureExtractor, MusicgenProcessor
  
  
  UpperCAmelCase_       :				List[str]		   =      random.Random()
  def        __SCREAMING_SNAKE_CASE    (      a__						:      str ,a__						:      Dict=1.0 ,a__						:      int=None ,a__						:      List[Any]=None					)				->							Optional[int]:
  	if rng is None:
  		__A   : Tuple								= global_rng
  
  	__A   : Optional[int]								= []
  	for batch_idx in range(shape[0]					):
  		values.append([]					)
  		for _ in range(shape[1]					):
  			values[-1].append(rng.random() * scale					)
  
  	return values
  
  
  
  
  
  
  
  @require_torch
  @require_sentencepiece
  class     lowerCamelCase_			(  unittest.TestCase  ):
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    int       ):
  		__A   : Union[str, Any]								= """facebook/musicgen-small"""
  		__A   : Optional[int]								= tempfile.mkdtemp()
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[str]				,			**__A	:    Optional[Any]       ):
  		return TaTokenizer.from_pretrained(self.checkpoint				,			**__A       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    int				,			**__A	:    Union[str, Any]       ):
  		return EncodecFeatureExtractor.from_pretrained(self.checkpoint				,			**__A       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[str]       ):
  		shutil.rmtree(self.tmpdirname       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Any       ):
  		__A   : Optional[Any]								= self.get_tokenizer()
  		__A   : Optional[Any]								= self.get_feature_extractor()
  
  		__A   : str								= MusicgenProcessor(tokenizer=__A				,			feature_extractor=__A       )
  
  		processor.save_pretrained(self.tmpdirname       )
  		__A   : List[Any]								= MusicgenProcessor.from_pretrained(self.tmpdirname       )
  
  		self.assertEqual(processor.tokenizer.get_vocab()				,			tokenizer.get_vocab()       )
  		self.assertIsInstance(processor.tokenizer				,			__A       )
  
  		self.assertEqual(processor.feature_extractor.to_json_string()				,			feature_extractor.to_json_string()       )
  		self.assertIsInstance(processor.feature_extractor				,			__A       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Any       ):
  		__A   : Optional[int]								= MusicgenProcessor(tokenizer=self.get_tokenizer()				,			feature_extractor=self.get_feature_extractor()       )
  		processor.save_pretrained(self.tmpdirname       )
  
  		__A   : Optional[Any]								= self.get_tokenizer(bos_token="""(BOS)"""				,			eos_token="""(EOS)"""       )
  		__A   : List[str]								= self.get_feature_extractor(do_normalize=__A				,			padding_value=1.0       )
  
  		__A   : Optional[Any]								= MusicgenProcessor.from_pretrained(
  		    self.tmpdirname				,			bos_token="""(BOS)"""				,			eos_token="""(EOS)"""				,			do_normalize=__A				,			padding_value=1.0       )
  
  		self.assertEqual(processor.tokenizer.get_vocab()				,			tokenizer_add_kwargs.get_vocab()       )
  		self.assertIsInstance(processor.tokenizer				,			__A       )
  
  		self.assertEqual(processor.feature_extractor.to_json_string()				,			feature_extractor_add_kwargs.to_json_string()       )
  		self.assertIsInstance(processor.feature_extractor				,			__A       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
  		__A   : int								= self.get_feature_extractor()
  		__A   : List[str]								= self.get_tokenizer()
  
  		__A   : Tuple								= MusicgenProcessor(tokenizer=__A				,			feature_extractor=__A       )
  
  		__A   : Optional[Any]								= floats_list((3, 1000)       )
  
  		__A   : int								= feature_extractor(__A				,			return_tensors="""np"""       )
  		__A   : int								= processor(__A				,			return_tensors="""np"""       )
  
  		for key in input_feat_extract.keys():
  			self.assertAlmostEqual(input_feat_extract[key].sum()				,			input_processor[key].sum()				,			delta=1e-2       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Union[str, Any]       ):
  		__A   : Dict								= self.get_feature_extractor()
  		__A   : int								= self.get_tokenizer()
  
  		__A   : Dict								= MusicgenProcessor(tokenizer=__A				,			feature_extractor=__A       )
  
  		__A   : Optional[int]								= """This is a test string"""
  
  		__A   : List[str]								= processor(text=__A       )
  
  		__A   : List[str]								= tokenizer(__A       )
  
  		for key in encoded_tok.keys():
  			self.assertListEqual(encoded_tok[key]				,			encoded_processor[key]       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Any       ):
  		__A   : List[Any]								= self.get_feature_extractor()
  		__A   : str								= self.get_tokenizer()
  
  		__A   : Optional[int]								= MusicgenProcessor(tokenizer=__A				,			feature_extractor=__A       )
  
  		__A   : Optional[int]								= [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]
  
  		__A   : List[str]								= processor.batch_decode(sequences=__A       )
  		__A   : Any								= tokenizer.batch_decode(__A       )
  
  		self.assertListEqual(__A				,			__A       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    int       ):
  		__A   : Optional[Any]								= self.get_feature_extractor()
  		__A   : Optional[Any]								= self.get_tokenizer()
  
  		__A   : Union[str, Any]								= MusicgenProcessor(tokenizer=__A				,			feature_extractor=__A       )
  
  		self.assertListEqual(
  		    processor.model_input_names				,			feature_extractor.model_input_names				,			msg="""`processor` and `feature_extractor` model input names do not match"""				,			)
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
  		__A   : List[str]								= self.get_feature_extractor(padding_side="""left"""       )
  		__A   : Optional[Any]								= self.get_tokenizer()
  
  		__A   : Tuple								= MusicgenProcessor(tokenizer=__A				,			feature_extractor=__A       )
  
  		__A   : str								= [floats_list((1, x)       )[0] for x in range(5				,			20				,			5       )]
  		__A   : List[str]								= processor(__A       ).padding_mask
  
  		__A   : Dict								= np.asarray(floats_list((3, 20)       )       )[:, None, :]
  		__A   : List[str]								= processor.batch_decode(__A				,			padding_mask=__A       )
  
  		self.assertIsInstance(__A				,			__A       )
  
  		for audio in decoded_audios:
  			self.assertIsInstance(__A				,			np.ndarray       )
  
  		self.assertTrue(decoded_audios[0].shape == (1, 10)       )
  		self.assertTrue(decoded_audios[1].shape == (1, 15)       )
  		self.assertTrue(decoded_audios[2].shape == (1, 20)       )
  
  
  
  
  
  
style_context: 
  
  import os
  import tempfile
  import unittest
  from typing import List
  
  from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
  from transformers.testing_utils import require_tokenizers
  from transformers.tokenization_utils import PreTrainedTokenizer
  from transformers.tokenization_utils_base import PreTrainedTokenizerBase
  
  
  
  
  
  
  @require_tokenizers
  class     lowerCamelCase_			(  unittest.TestCase  ):
  	_lowercase		:      List[str]    				=							EsmTokenizer
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
  		super().setUp()
  		__A   : Dict								= tempfile.mkdtemp()
  		# fmt: off
  		__A   : List[str]								= ["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
  		# fmt: on
  		__A   : int								= os.path.join(self.tmpdirname				,			VOCAB_FILES_NAMES["""vocab_file"""]       )
  		with open(self.vocab_file				,			"""w"""				,			encoding="""utf-8"""       ) as vocab_writer:
  			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Dict				,			**__A	:    int       ):
  		return [self.get_tokenizer(**__A       )]
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Optional[int]				,			**__A	:    List[Any]       ):
  		return self.tokenizer_class.from_pretrained(self.tmpdirname				,			**__A       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[str]       ):
  		__A   : Union[str, Any]								= self.tokenizer_class(self.vocab_file       )
  
  		__A   : int								= tokenizer.tokenize("""LAGVS"""       )
  		self.assertListEqual(__A				,			["""L""", """A""", """G""", """V""", """S"""]       )
  		self.assertListEqual(tokenizer.convert_tokens_to_ids(__A       )				,			[4, 5, 6, 7, 8]       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    str       ):
  		__A   : Union[str, Any]								= self.tokenizer_class(self.vocab_file       )
  
  		__A   : int								= """LAGVS"""
  		self.assertListEqual(tokenizer.encode(__A       )				,			[0, 4, 5, 6, 7, 8, 2]       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    Dict       ):
  		__A   : str								= self.tokenizer_class(self.vocab_file       )
  
  		__A   : Tuple								= ["""LAGVS""", """WCB"""]
  		__A   : str								= tokenizer(__A				,			padding=__A       )["""input_ids"""]
  
  		self.assertListEqual(__A				,			[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2]]       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
  		__A   : Dict								= self.tokenizer_class(self.vocab_file       )
  
  		__A   : Any								= ["""LAGVS""", """WCB"""]
  		__A   : str								= tokenizer(__A				,			padding=__A       )["""input_ids"""]
  
  		self.assertListEqual(__A				,			[[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2, 1, 1]]       )
  
  
  
  
  	def 					lowerCAmelCase_					(    self	:    List[str]       ):
  		__A   : Optional[int]								= self.get_tokenizers(fast=__A       )
  		for tokenizer in tokenizers:
  			with self.subTest(F"""{tokenizer.__class__.__name__}"""       ):
  				__A   : Any								= """<unk>"""
  				__A   : str								= """<mask>"""
  
  				__A   : Optional[Any]								= tokenizer.tokenize(__A       )
  				__A   : str								= tokenizer.tokenize(__A       )
  
  				self.assertEqual(len(__A       )				,			1       )
  				self.assertEqual(len(__A       )				,			1       )
  				self.assertEqual(token_a[0]				,			__A       )
  				self.assertEqual(token_a[0]				,			__A       )
  
  
  
  
  
  