code: 
  
  
  
  """simple docstring"""
  
  
  
  
  import random
  import shutil
  import tempfile
  import unittest
  
  import numpy as np
  
  from transformers import TaTokenizer, TaTokenizerFast
  from transformers.testing_utils import require_sentencepiece, require_torch
  from transformers.utils.import_utils import is_speech_available, is_torch_available
  
  
  if is_torch_available():
   pass
  
  if is_speech_available():
   from transformers import EncodecFeatureExtractor, MusicgenProcessor
  
  
  _lowerCAmelCase       :      List[str]							=							random.Random()
  
  def 		__snake_case		(   SCREAMING_SNAKE_CASE__			:		Union[str, Any]		,						SCREAMING_SNAKE_CASE__			:		Tuple=1.0		,						SCREAMING_SNAKE_CASE__			:		int=None		,						SCREAMING_SNAKE_CASE__			:		Any=None       )						->     List[str]:
  
  
  
  
  
       '''simple docstring'''
  
       if rng is None:
            _UpperCAmelCase    :			Any              =					global_rng
  
       _UpperCAmelCase    :			List[Any]              =					[]
       for batch_idx in range(shape[0]       ):
            values.append([]       )
            for _ in range(shape[1]       ):
                 values[-1].append(rng.random() * scale       )
  
       return values
  
  
  
  
  
  
  @require_torch
  @require_sentencepiece
  class 			UpperCAmelCase_						(       unittest.TestCase		):
  
  
  
        def 		snake_case_      (  self  :			Union[str, Any]		):
             _UpperCAmelCase    :			Dict              =					"facebook/musicgen-small"
             _UpperCAmelCase    :			Optional[Any]              =					tempfile.mkdtemp()
  
  
  
        def 		snake_case_      (  self  :			int  ,			**A  :			str		):
             return TaTokenizer.from_pretrained(self.checkpoint  ,			**A		)
  
  
  
        def 		snake_case_      (  self  :			Optional[int]  ,			**A  :			Union[str, Any]		):
             return EncodecFeatureExtractor.from_pretrained(self.checkpoint  ,			**A		)
  
  
  
        def 		snake_case_      (  self  :			List[Any]		):
             shutil.rmtree(self.tmpdirname		)
  
  
  
        def 		snake_case_      (  self  :			List[Any]		):
             _UpperCAmelCase    :			List[str]              =					self.get_tokenizer()
             _UpperCAmelCase    :			Union[str, Any]              =					self.get_feature_extractor()
  
             _UpperCAmelCase    :			List[Any]              =					MusicgenProcessor(tokenizer=A  ,			feature_extractor=A		)
  
             processor.save_pretrained(self.tmpdirname		)
             _UpperCAmelCase    :			Optional[int]              =					MusicgenProcessor.from_pretrained(self.tmpdirname		)
  
             self.assertEqual(processor.tokenizer.get_vocab()  ,			tokenizer.get_vocab()		)
             self.assertIsInstance(processor.tokenizer  ,			A		)
  
             self.assertEqual(processor.feature_extractor.to_json_string()  ,			feature_extractor.to_json_string()		)
             self.assertIsInstance(processor.feature_extractor  ,			A		)
  
  
  
        def 		snake_case_      (  self  :			int		):
             _UpperCAmelCase    :			Dict              =					MusicgenProcessor(tokenizer=self.get_tokenizer()  ,			feature_extractor=self.get_feature_extractor()		)
             processor.save_pretrained(self.tmpdirname		)
  
             _UpperCAmelCase    :			int              =					self.get_tokenizer(bos_token="(BOS)"  ,			eos_token="(EOS)"		)
             _UpperCAmelCase    :			Dict              =					self.get_feature_extractor(do_normalize=A  ,			padding_value=1.0		)
  
             _UpperCAmelCase    :			Optional[int]              =					MusicgenProcessor.from_pretrained(
                 self.tmpdirname  ,			bos_token="(BOS)"  ,			eos_token="(EOS)"  ,			do_normalize=A  ,			padding_value=1.0		)
  
             self.assertEqual(processor.tokenizer.get_vocab()  ,			tokenizer_add_kwargs.get_vocab()		)
             self.assertIsInstance(processor.tokenizer  ,			A		)
  
             self.assertEqual(processor.feature_extractor.to_json_string()  ,			feature_extractor_add_kwargs.to_json_string()		)
             self.assertIsInstance(processor.feature_extractor  ,			A		)
  
  
  
        def 		snake_case_      (  self  :			List[Any]		):
             _UpperCAmelCase    :			Any              =					self.get_feature_extractor()
             _UpperCAmelCase    :			Tuple              =					self.get_tokenizer()
  
             _UpperCAmelCase    :			str              =					MusicgenProcessor(tokenizer=A  ,			feature_extractor=A		)
  
             _UpperCAmelCase    :			Any              =					floats_list((3, 1_0_0_0)		)
  
             _UpperCAmelCase    :			Any              =					feature_extractor(A  ,			return_tensors="np"		)
             _UpperCAmelCase    :			Optional[int]              =					processor(A  ,			return_tensors="np"		)
  
             for key in input_feat_extract.keys():
                  self.assertAlmostEqual(input_feat_extract[key].sum()  ,			input_processor[key].sum()  ,			delta=1e-2		)
  
  
  
        def 		snake_case_      (  self  :			int		):
             _UpperCAmelCase    :			List[Any]              =					self.get_feature_extractor()
             _UpperCAmelCase    :			List[Any]              =					self.get_tokenizer()
  
             _UpperCAmelCase    :			Optional[int]              =					MusicgenProcessor(tokenizer=A  ,			feature_extractor=A		)
  
             _UpperCAmelCase    :			Optional[int]              =					"This is a test string"
  
             _UpperCAmelCase    :			List[str]              =					processor(text=A		)
  
             _UpperCAmelCase    :			Optional[int]              =					tokenizer(A		)
  
             for key in encoded_tok.keys():
                  self.assertListEqual(encoded_tok[key]  ,			encoded_processor[key]		)
  
  
  
        def 		snake_case_      (  self  :			str		):
             _UpperCAmelCase    :			List[str]              =					self.get_feature_extractor()
             _UpperCAmelCase    :			str              =					self.get_tokenizer()
  
             _UpperCAmelCase    :			str              =					MusicgenProcessor(tokenizer=A  ,			feature_extractor=A		)
  
             _UpperCAmelCase    :			str              =					[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]
  
             _UpperCAmelCase    :			Any              =					processor.batch_decode(sequences=A		)
             _UpperCAmelCase    :			str              =					tokenizer.batch_decode(A		)
  
             self.assertListEqual(A  ,			A		)
  
  
  
        def 		snake_case_      (  self  :			List[str]		):
             _UpperCAmelCase    :			Any              =					self.get_feature_extractor()
             _UpperCAmelCase    :			Optional[int]              =					self.get_tokenizer()
  
             _UpperCAmelCase    :			Optional[Any]              =					MusicgenProcessor(tokenizer=A  ,			feature_extractor=A		)
  
             self.assertListEqual(
                 processor.model_input_names  ,			feature_extractor.model_input_names  ,			msg="`processor` and `feature_extractor` model input names do not match"  ,			)
  
  
  
        def 		snake_case_      (  self  :			Optional[int]		):
             _UpperCAmelCase    :			str              =					self.get_feature_extractor(padding_side="left"		)
             _UpperCAmelCase    :			Union[str, Any]              =					self.get_tokenizer()
  
             _UpperCAmelCase    :			List[str]              =					MusicgenProcessor(tokenizer=A  ,			feature_extractor=A		)
  
             _UpperCAmelCase    :			int              =					[floats_list((1, x)		)[0] for x in range(5  ,			2_0  ,			5		)]
             _UpperCAmelCase    :			List[Any]              =					processor(A		).padding_mask
  
             _UpperCAmelCase    :			List[str]              =					np.asarray(floats_list((3, 2_0)		)		)[:, None, :]
             _UpperCAmelCase    :			str              =					processor.batch_decode(A  ,			padding_mask=A		)
  
             self.assertIsInstance(A  ,			A		)
  
             for audio in decoded_audios:
                  self.assertIsInstance(A  ,			np.ndarray		)
  
             self.assertTrue(decoded_audios[0].shape == (1, 1_0)		)
             self.assertTrue(decoded_audios[1].shape == (1, 1_5)		)
             self.assertTrue(decoded_audios[2].shape == (1, 2_0)		)
  
style_context: 
  
  
  
  """simple docstring"""
  
  
  
  
  from arguments import TokenizerTrainingArguments
  from datasets import load_dataset
  from tqdm import tqdm
  
  from transformers import AutoTokenizer, HfArgumentParser
  from transformers.models.gpta.tokenization_gpta import bytes_to_unicode
  
  def 		__snake_case		(   SCREAMING_SNAKE_CASE__			:		str=10       )						->     List[Any]:
  
  
  
  
  
       '''simple docstring'''
  
       for _ in tqdm(range(0		,						args.n_examples		,						SCREAMING_SNAKE_CASE__       )       ):
            yield [next(SCREAMING_SNAKE_CASE__       )[args.text_column] for _ in range(SCREAMING_SNAKE_CASE__       )]
  
  
  # Configuration
  _lowerCAmelCase       :      Optional[Any]							=							HfArgumentParser(TokenizerTrainingArguments)
  _lowerCAmelCase       :      Dict							=							parser.parse_args()
  
  # Base tokenizer
  _lowerCAmelCase       :      int							=							AutoTokenizer.from_pretrained(args.base_tokenizer)
  _lowerCAmelCase       :      List[str]							=							list(bytes_to_unicode().values())
  
  # Load dataset
  _lowerCAmelCase       :      Dict							=							load_dataset(args.dataset_name, split="train", streaming=True)
  _lowerCAmelCase       :      List[Any]							=							iter(dataset)
  
  
  # Training and saving
  _lowerCAmelCase       :      List[str]							=							tokenizer.train_new_from_iterator(
      batch_iterator(), vocab_size=args.vocab_size, initial_alphabet=base_vocab
  )
  new_tokenizer.save_pretrained(args.tokenizer_name, push_to_hub=args.push_to_hub)
  