code: from typing import List, Optional, Union
  
  import numpy as np
  import torch
  import torchaudio.compliance.kaldi as ta_kaldi
  
  from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
  from ...feature_extraction_utils import BatchFeature
  from ...utils import TensorType, logging
  
  
  a     :			Dict					    =						logging.get_logger(__name__)
  
  class 				_lowercase						(    UpperCAmelCase__ ):
  
  
  
  
  
  
      '''simple docstring'''
      SCREAMING_SNAKE_CASE:							Tuple        =						['input_values', 'attention_mask']
  
  
  
  
      def __init__(						self					,	lowerCamelCase__=1					,	lowerCamelCase__=16_000					,	lowerCamelCase__=128					,	lowerCamelCase__=1_024					,	lowerCamelCase__=0.0					,	lowerCamelCase__=True					,	lowerCamelCase__=-4.2_6_7_7_3_9_3					,	lowerCamelCase__=4.5_6_8_9_9_7_4					,	lowerCamelCase__=False					,	**lowerCamelCase__					,	):
         super().__init__(feature_size=lowerCamelCase__					,	sampling_rate=lowerCamelCase__					,	padding_value=lowerCamelCase__					,	**lowerCamelCase__   )
         lowerCAmelCase_:       Tuple         =  num_mel_bins
         lowerCAmelCase_:       Optional[int]         =  max_length
         lowerCAmelCase_:       Dict         =  do_normalize
         lowerCAmelCase_:       Optional[Any]         =  mean
         lowerCAmelCase_:       Dict         =  std
         lowerCAmelCase_:       Dict         =  return_attention_mask
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__					,	lowerCamelCase__					,	):
         lowerCAmelCase_:       Union[str, Any]         =  torch.from_numpy(lowerCamelCase__   ).unsqueeze(0   )
         lowerCAmelCase_:       Dict         =  ta_kaldi.fbank(
             lowerCamelCase__					,	htk_compat=lowerCamelCase__					,	sample_frequency=self.sampling_rate					,	use_energy=lowerCamelCase__					,	window_type="hanning"					,	num_mel_bins=self.num_mel_bins					,	dither=0.0					,	frame_shift=10					,	)
  
         lowerCAmelCase_:       Optional[int]         =  fbank.shape[0]
         lowerCAmelCase_:       Tuple         =  max_length - n_frames
  
         # pad or truncate, depending on difference
         if difference > 0:
            lowerCAmelCase_:       int         =  torch.nn.ZeroPadad((0, 0, 0, difference)   )
            lowerCAmelCase_:       int         =  pad_module(lowerCamelCase__   )
         elif difference < 0:
            lowerCAmelCase_:       int         =  fbank[0:max_length, :]
  
         lowerCAmelCase_:       List[Any]         =  fbank.numpy()
  
         return fbank
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__   ):
         return (input_values - (self.mean)) / (self.std * 2)
  
  
  
  
      def __call__(						self					,	lowerCamelCase__					,	lowerCamelCase__ = None					,	lowerCamelCase__ = None					,	**lowerCamelCase__					,	):
  
         if sampling_rate is not None:
            if sampling_rate != self.sampling_rate:
               raise ValueError(
                   F'''The model corresponding to this feature extractor: {self} was trained using a sampling rate of'''
                   F''' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'''
                   F''' {self.sampling_rate} and not {sampling_rate}.'''   )
         else:
            logger.warning(
                "It is strongly recommended to pass the `sampling_rate` argument to this function. "
                "Failing to do so can result in silent errors that might be hard to debug."   )
  
         lowerCAmelCase_:       List[str]         =  isinstance(lowerCamelCase__					,	np.ndarray   ) and len(raw_speech.shape   ) > 1
         if is_batched_numpy and len(raw_speech.shape   ) > 2:
            raise ValueError(F'''Only mono-channel audio is supported for input to {self}'''   )
         lowerCAmelCase_:       Tuple         =  is_batched_numpy or (
             isinstance(lowerCamelCase__					,	(list, tuple)   ) and (isinstance(raw_speech[0]					,	(np.ndarray, tuple, list)   ))
         )
  
         if is_batched:
            lowerCAmelCase_:       Any         =  [np.asarray(lowerCamelCase__					,	dtype=np.floataa   ) for speech in raw_speech]
         elif not is_batched and not isinstance(lowerCamelCase__					,	np.ndarray   ):
            lowerCAmelCase_:       List[str]         =  np.asarray(lowerCamelCase__					,	dtype=np.floataa   )
         elif isinstance(lowerCamelCase__					,	np.ndarray   ) and raw_speech.dtype is np.dtype(np.floataa   ):
            lowerCAmelCase_:       Optional[Any]         =  raw_speech.astype(np.floataa   )
  
         # always return batch
         if not is_batched:
            lowerCAmelCase_:       int         =  [raw_speech]
  
         # extract fbank features and pad/truncate to max_length
         lowerCAmelCase_:       List[Any]         =  [self._extract_fbank_features(lowerCamelCase__					,	max_length=self.max_length   ) for waveform in raw_speech]
  
         # convert into BatchFeature
         lowerCAmelCase_:       Optional[int]         =  BatchFeature({"input_values": features}   )
  
         # make sure list is in array format
         lowerCAmelCase_:       Tuple         =  padded_inputs.get("input_values"   )
         if isinstance(input_values[0]					,	lowerCamelCase__   ):
            lowerCAmelCase_:       str         =  [np.asarray(lowerCamelCase__					,	dtype=np.floataa   ) for feature in input_values]
  
         # normalization
         if self.do_normalize:
            lowerCAmelCase_:       int         =  [self.normalize(lowerCamelCase__   ) for feature in input_values]
  
         if return_tensors is not None:
            lowerCAmelCase_:       Union[str, Any]         =  padded_inputs.convert_to_tensors(lowerCamelCase__   )
  
         return padded_inputs
style_context: import tempfile
  import unittest
  
  import numpy as np
  
  import transformers
  from transformers import GPTaConfig, GPTaTokenizer, is_flax_available, is_torch_available
  from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow
  
  from ...generation.test_flax_utils import FlaxGenerationTesterMixin
  from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
  
  
  if is_flax_available():
   import jax
   import jax.numpy as jnp
  
   from transformers.modeling_flax_pytorch_utils import (
       convert_pytorch_state_dict_to_flax,
       load_flax_weights_in_pytorch_model,
   )
   from transformers.models.gpta.modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel
  
  if is_torch_available():
   import torch
  
  class 				_lowercase						:
  
  
  
  
  
  
      '''simple docstring'''
  
  
  
  
      def __init__(						self					,	lowerCamelCase__					,	lowerCamelCase__=14					,	lowerCamelCase__=7					,	lowerCamelCase__=True					,	lowerCamelCase__=True					,	lowerCamelCase__=False					,	lowerCamelCase__=True					,	lowerCamelCase__=99					,	lowerCamelCase__=32					,	lowerCamelCase__=5					,	lowerCamelCase__=4					,	lowerCamelCase__=37					,	lowerCamelCase__="gelu"					,	lowerCamelCase__=0.1					,	lowerCamelCase__=0.1					,	lowerCamelCase__=512					,	lowerCamelCase__=0.0_2					,	):
         lowerCAmelCase_:       int         =  parent
         lowerCAmelCase_:       Tuple         =  batch_size
         lowerCAmelCase_:       int         =  seq_length
         lowerCAmelCase_:       Any         =  is_training
         lowerCAmelCase_:       Any         =  use_input_mask
         lowerCAmelCase_:       Optional[Any]         =  use_token_type_ids
         lowerCAmelCase_:       str         =  use_labels
         lowerCAmelCase_:       Tuple         =  vocab_size
         lowerCAmelCase_:       List[Any]         =  hidden_size
         lowerCAmelCase_:       Optional[int]         =  num_hidden_layers
         lowerCAmelCase_:       List[str]         =  num_attention_heads
         lowerCAmelCase_:       int         =  intermediate_size
         lowerCAmelCase_:       Tuple         =  hidden_act
         lowerCAmelCase_:       Dict         =  hidden_dropout_prob
         lowerCAmelCase_:       Dict         =  attention_probs_dropout_prob
         lowerCAmelCase_:       Any         =  max_position_embeddings
         lowerCAmelCase_:       Dict         =  initializer_range
         lowerCAmelCase_:       List[Any]         =  None
         lowerCAmelCase_:       List[Any]         =  vocab_size - 1
         lowerCAmelCase_:       Union[str, Any]         =  vocab_size - 1
         lowerCAmelCase_:       int         =  vocab_size - 1
  
  
  
  
      def     _a						(						self   ):
         lowerCAmelCase_:       Tuple         =  ids_tensor([self.batch_size, self.seq_length]					,	self.vocab_size   )
  
         lowerCAmelCase_:       Any         =  None
         if self.use_input_mask:
            lowerCAmelCase_:       Union[str, Any]         =  random_attention_mask([self.batch_size, self.seq_length]   )
  
         lowerCAmelCase_:       Union[str, Any]         =  GPTaConfig(
             vocab_size=self.vocab_size					,	n_embd=self.hidden_size					,	n_layer=self.num_hidden_layers					,	n_head=self.num_attention_heads					,	n_positions=self.max_position_embeddings					,	use_cache=lowerCamelCase__					,	bos_token_id=self.bos_token_id					,	eos_token_id=self.eos_token_id					,	pad_token_id=self.pad_token_id					,	)
  
         return (config, input_ids, input_mask)
  
  
  
  
      def     _a						(						self   ):
         lowerCAmelCase_:       str         =  self.prepare_config_and_inputs()
         lowerCAmelCase_     ,  lowerCAmelCase_     ,  lowerCAmelCase_:       Dict         =  config_and_inputs
         lowerCAmelCase_:       List[str]         =  {"input_ids": input_ids, "attention_mask": attention_mask}
         return config, inputs_dict
  
  
  
  
      def     _a						(						self   ):
         lowerCAmelCase_     ,  lowerCAmelCase_     ,  lowerCAmelCase_:       str         =  self.prepare_config_and_inputs()
  
         lowerCAmelCase_:       List[str]         =  floats_tensor([self.batch_size, self.seq_length, self.hidden_size]   )
         lowerCAmelCase_:       Tuple         =  ids_tensor([self.batch_size, self.seq_length]					,	vocab_size=2   )
  
         return (
             config,
             input_ids,
             attention_mask,
             encoder_hidden_states,
             encoder_attention_mask,
         )
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__   ):
         lowerCAmelCase_:       Optional[Any]         =  20
         lowerCAmelCase_:       Union[str, Any]         =  model_class_name(lowerCamelCase__   )
  
         lowerCAmelCase_:       List[str]         =  model.init_cache(input_ids.shape[0]					,	lowerCamelCase__   )
         lowerCAmelCase_:       Any         =  jnp.ones((input_ids.shape[0], max_decoder_length)					,	dtype="i4"   )
  
         lowerCAmelCase_:       Union[str, Any]         =  jnp.broadcast_to(
             jnp.arange(input_ids.shape[-1] - 1   )[None, :]					,	(input_ids.shape[0], input_ids.shape[-1] - 1)   )
         lowerCAmelCase_:       Optional[Any]         =  model(
             input_ids[:, :-1]					,	attention_mask=lowerCamelCase__					,	past_key_values=lowerCamelCase__					,	position_ids=lowerCamelCase__					,	)
  
         lowerCAmelCase_:       Any         =  jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]					,	dtype="i4"   )
         lowerCAmelCase_:       Optional[int]         =  model(
             input_ids[:, -1:]					,	attention_mask=lowerCamelCase__					,	past_key_values=outputs_cache.past_key_values					,	position_ids=lowerCamelCase__					,	)
  
         lowerCAmelCase_:       List[Any]         =  model(lowerCamelCase__   )
  
         lowerCAmelCase_:       List[Any]         =  np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])   )   )
         self.parent.assertTrue(diff < 1E-3					,	msg=F'''Max diff is {diff}'''   )
  
  
  
  
      def     _a						(						self					,	lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__   ):
         lowerCAmelCase_:       Union[str, Any]         =  20
         lowerCAmelCase_:       List[str]         =  model_class_name(lowerCamelCase__   )
  
         lowerCAmelCase_:       Any         =  jnp.concatenate(
             [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])   )]					,	axis=-1					,	)
  
         lowerCAmelCase_:       str         =  model.init_cache(input_ids.shape[0]					,	lowerCamelCase__   )
         lowerCAmelCase_:       List[Any]         =  jnp.broadcast_to(
             jnp.arange(input_ids.shape[-1] - 1   )[None, :]					,	(input_ids.shape[0], input_ids.shape[-1] - 1)   )
  
         lowerCAmelCase_:       List[Any]         =  model(
             input_ids[:, :-1]					,	attention_mask=lowerCamelCase__					,	past_key_values=lowerCamelCase__					,	position_ids=lowerCamelCase__					,	)
         lowerCAmelCase_:       Dict         =  jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]					,	dtype="i4"   )
         lowerCAmelCase_:       Dict         =  model(
             input_ids[:, -1:]					,	past_key_values=outputs_cache.past_key_values					,	attention_mask=lowerCamelCase__					,	position_ids=lowerCamelCase__					,	)
  
         lowerCAmelCase_:       Union[str, Any]         =  model(lowerCamelCase__					,	attention_mask=lowerCamelCase__   )
  
         lowerCAmelCase_:       List[str]         =  np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])   )   )
         self.parent.assertTrue(diff < 1E-3					,	msg=F'''Max diff is {diff}'''   )
  
  
  
  
  
  @require_flax
  class 				_lowercase						(    UpperCAmelCase__   ,     UpperCAmelCase__   ,     unittest.TestCase ):
  
  
  
  
  
  
      '''simple docstring'''
      SCREAMING_SNAKE_CASE:							Optional[int]        =						(FlaxGPTaModel, FlaxGPTaLMHeadModel) if is_flax_available() else ()
      SCREAMING_SNAKE_CASE:							Union[str, Any]        =						(FlaxGPTaLMHeadModel,) if is_flax_available() else ()
  
  
  
  
      def     _a						(						self   ):
         lowerCAmelCase_:       Any         =  FlaxGPTaModelTester(self   )
  
  
  
  
      def     _a						(						self   ):
         for model_class_name in self.all_model_classes:
            lowerCAmelCase_     ,  lowerCAmelCase_     ,  lowerCAmelCase_:       Dict         =  self.model_tester.prepare_config_and_inputs()
            self.model_tester.check_use_cache_forward(lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__   )
  
  
  
  
      def     _a						(						self   ):
         for model_class_name in self.all_model_classes:
            lowerCAmelCase_     ,  lowerCAmelCase_     ,  lowerCAmelCase_:       Optional[Any]         =  self.model_tester.prepare_config_and_inputs()
            self.model_tester.check_use_cache_forward_with_attn_mask(
                lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__					,	lowerCamelCase__   )
  
  
  
  
      @slow
      def     _a						(						self   ):
         lowerCAmelCase_:       Any         =  GPTaTokenizer.from_pretrained("gpt2"					,	pad_token="</s>"					,	padding_side="left"   )
         lowerCAmelCase_:       Optional[int]         =  tokenizer(["Hello this is a long string", "Hey"]					,	return_tensors="np"					,	padding=lowerCamelCase__					,	truncation=lowerCamelCase__   )
  
         lowerCAmelCase_:       List[Any]         =  FlaxGPTaLMHeadModel.from_pretrained("gpt2"   )
         lowerCAmelCase_:       List[str]         =  False
         lowerCAmelCase_:       int         =  model.config.eos_token_id
  
         lowerCAmelCase_:       Optional[int]         =  jax.jit(model.generate   )
  
         lowerCAmelCase_:       Any         =  jit_generate(inputs["input_ids"]					,	attention_mask=inputs["attention_mask"]   ).sequences
  
         lowerCAmelCase_:       Any         =  tokenizer.batch_decode(lowerCamelCase__					,	skip_special_tokens=lowerCamelCase__   )
  
         lowerCAmelCase_:       Dict         =  [
             "Hello this is a long string of words. I'm going to start with the first one.\n",
             "Hey, I'm not sure if I'm going to be able to do",
         ]
  
         self.assertListEqual(lowerCamelCase__					,	lowerCamelCase__   )
  
  
  
  
      @is_pt_flax_cross_test
      def     _a						(						self   ):
         lowerCAmelCase_     ,  lowerCAmelCase_:       Union[str, Any]         =  self.model_tester.prepare_config_and_inputs_for_common()
  
         for model_class in self.all_model_classes:
            with self.subTest(model_class.__name__   ):
               # prepare inputs
               lowerCAmelCase_:       Any         =  self._prepare_for_class(lowerCamelCase__					,	lowerCamelCase__   )
               lowerCAmelCase_:       str         =  {k: torch.tensor(v.tolist()   ) for k, v in prepared_inputs_dict.items()}
  
               # load corresponding PyTorch class
               lowerCAmelCase_:       Dict         =  model_class.__name__[4:]  # Skip the "Flax" at the beginning
               lowerCAmelCase_:       Union[str, Any]         =  getattr(lowerCamelCase__					,	lowerCamelCase__   )
  
               lowerCAmelCase_     ,  lowerCAmelCase_:       str         =  pt_inputs["input_ids"].shape
               lowerCAmelCase_:       str         =  np.random.randint(0					,	seq_length - 1					,	size=(batch_size,)   )
               for batch_idx, start_index in enumerate(lowerCamelCase__   ):
                  lowerCAmelCase_:       List[str]         =  0
                  lowerCAmelCase_:       int         =  1
                  lowerCAmelCase_:       List[Any]         =  0
                  lowerCAmelCase_:       List[Any]         =  1
               lowerCAmelCase_:       Tuple         =  pt_model_class(lowerCamelCase__   ).eval()
               lowerCAmelCase_:       Tuple         =  model_class(lowerCamelCase__					,	dtype=jnp.floataa   )
  
               lowerCAmelCase_:       int         =  convert_pytorch_state_dict_to_flax(pt_model.state_dict()					,	lowerCamelCase__   )
               lowerCAmelCase_:       Optional[int]         =  fx_state
  
               with torch.no_grad():
                  lowerCAmelCase_:       Union[str, Any]         =  pt_model(**lowerCamelCase__   ).to_tuple()
  
               lowerCAmelCase_:       List[str]         =  fx_model(**lowerCamelCase__   ).to_tuple()
               self.assertEqual(len(lowerCamelCase__   )					,	len(lowerCamelCase__   )					,	"Output lengths differ between Flax and PyTorch"   )
               for fx_output, pt_output in zip(lowerCamelCase__					,	lowerCamelCase__   ):
                  self.assert_almost_equals(fx_output[:, -1]					,	pt_output[:, -1].numpy()					,	4E-2   )
  
               with tempfile.TemporaryDirectory() as tmpdirname:
                  pt_model.save_pretrained(lowerCamelCase__   )
                  lowerCAmelCase_:       Dict         =  model_class.from_pretrained(lowerCamelCase__					,	from_pt=lowerCamelCase__   )
  
               lowerCAmelCase_:       List[Any]         =  fx_model_loaded(**lowerCamelCase__   ).to_tuple()
               self.assertEqual(
                   len(lowerCamelCase__   )					,	len(lowerCamelCase__   )					,	"Output lengths differ between Flax and PyTorch"   )
               for fx_output_loaded, pt_output in zip(lowerCamelCase__					,	lowerCamelCase__   ):
                  self.assert_almost_equals(fx_output_loaded[:, -1]					,	pt_output[:, -1].numpy()					,	4E-2   )
  
  
  
  
      @is_pt_flax_cross_test
      def     _a						(						self   ):
         lowerCAmelCase_     ,  lowerCAmelCase_:       List[str]         =  self.model_tester.prepare_config_and_inputs_for_common()
         for model_class in self.all_model_classes:
            with self.subTest(model_class.__name__   ):
               # prepare inputs
               lowerCAmelCase_:       Any         =  self._prepare_for_class(lowerCamelCase__					,	lowerCamelCase__   )
               lowerCAmelCase_:       int         =  {k: torch.tensor(v.tolist()   ) for k, v in prepared_inputs_dict.items()}
  
               # load corresponding PyTorch class
               lowerCAmelCase_:       List[str]         =  model_class.__name__[4:]  # Skip the "Flax" at the beginning
               lowerCAmelCase_:       Dict         =  getattr(lowerCamelCase__					,	lowerCamelCase__   )
  
               lowerCAmelCase_:       Dict         =  pt_model_class(lowerCamelCase__   ).eval()
               lowerCAmelCase_:       Optional[Any]         =  model_class(lowerCamelCase__					,	dtype=jnp.floataa   )
  
               lowerCAmelCase_:       Optional[Any]         =  load_flax_weights_in_pytorch_model(lowerCamelCase__					,	fx_model.params   )
               lowerCAmelCase_     ,  lowerCAmelCase_:       List[str]         =  pt_inputs["input_ids"].shape
               lowerCAmelCase_:       Optional[Any]         =  np.random.randint(0					,	seq_length - 1					,	size=(batch_size,)   )
               for batch_idx, start_index in enumerate(lowerCamelCase__   ):
                  lowerCAmelCase_:       Dict         =  0
                  lowerCAmelCase_:       List[Any]         =  1
                  lowerCAmelCase_:       Dict         =  0
                  lowerCAmelCase_:       Optional[int]         =  1
  
               # make sure weights are tied in PyTorch
               pt_model.tie_weights()
  
               with torch.no_grad():
                  lowerCAmelCase_:       Any         =  pt_model(**lowerCamelCase__   ).to_tuple()
  
               lowerCAmelCase_:       Union[str, Any]         =  fx_model(**lowerCamelCase__   ).to_tuple()
               self.assertEqual(len(lowerCamelCase__   )					,	len(lowerCamelCase__   )					,	"Output lengths differ between Flax and PyTorch"   )
               for fx_output, pt_output in zip(lowerCamelCase__					,	lowerCamelCase__   ):
                  self.assert_almost_equals(fx_output[:, -1]					,	pt_output[:, -1].numpy()					,	4E-2   )
  
               with tempfile.TemporaryDirectory() as tmpdirname:
                  fx_model.save_pretrained(lowerCamelCase__   )
                  lowerCAmelCase_:       str         =  pt_model_class.from_pretrained(lowerCamelCase__					,	from_flax=lowerCamelCase__   )
  
               with torch.no_grad():
                  lowerCAmelCase_:       int         =  pt_model_loaded(**lowerCamelCase__   ).to_tuple()
  
               self.assertEqual(
                   len(lowerCamelCase__   )					,	len(lowerCamelCase__   )					,	"Output lengths differ between Flax and PyTorch"   )
               for fx_output, pt_output in zip(lowerCamelCase__					,	lowerCamelCase__   ):
                  self.assert_almost_equals(fx_output[:, -1]					,	pt_output[:, -1].numpy()					,	4E-2   )
  
  
  
  
      @slow
      def     _a						(						self   ):
         for model_class_name in self.all_model_classes:
            lowerCAmelCase_:       Tuple         =  model_class_name.from_pretrained("gpt2"					,	from_pt=lowerCamelCase__   )
            lowerCAmelCase_:       int         =  model(np.ones((1, 1)   )   )
            self.assertIsNotNone(lowerCamelCase__   )