


import unittest

from transformers import SpeechaTextaConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers.models.speech_to_text_a.modeling_speech_to_text_a import (
           SpeechaTextaDecoder,
           SpeechaTextaForCausalLM,
       )


@require_torch
class 	a__						:

       def __init__( self   :  Dict   ,				lowerCamelCase_   :  Any   ,				lowerCamelCase_   :  Dict=9_9   ,				lowerCamelCase_   :  List[Any]=1_3   ,				lowerCamelCase_   :  Dict=1_6   ,				lowerCamelCase_   :  Any=7   ,				lowerCamelCase_   :  Dict=True   ,				lowerCamelCase_   :  List[str]=True   ,				lowerCamelCase_   :  Tuple=True   ,				lowerCamelCase_   :  Any=False   ,				lowerCamelCase_   :  Union[str, Any]=True   ,				lowerCamelCase_   :  int=2   ,				lowerCamelCase_   :  Tuple=3_2   ,				lowerCamelCase_   :  int=4   ,				lowerCamelCase_   :  Dict=4   ,				lowerCamelCase_   :  str=3_0   ,				lowerCamelCase_   :  Optional[int]=0   ,				lowerCamelCase_   :  Optional[Any]=1   ,				lowerCamelCase_   :  Any=2   ,				lowerCamelCase_   :  Optional[int]=None   ,				):
            a_     :  Optional[Any]				      =     parent
            a_     :  Union[str, Any]				      =     batch_size
            a_     :  Tuple				      =     decoder_seq_length
            # For common tests
            a_     :  Union[str, Any]				      =     self.decoder_seq_length
            a_     :  List[str]				      =     is_training
            a_     :  int				      =     use_attention_mask
            a_     :  Dict				      =     use_labels

            a_     :  List[str]				      =     vocab_size
            a_     :  Optional[int]				      =     d_model
            a_     :  str				      =     d_model
            a_     :  Any				      =     decoder_layers
            a_     :  Tuple				      =     decoder_layers
            a_     :  Dict				      =     decoder_ffn_dim
            a_     :  List[str]				      =     decoder_attention_heads
            a_     :  List[Any]				      =     decoder_attention_heads
            a_     :  List[str]				      =     eos_token_id
            a_     :  Any				      =     bos_token_id
            a_     :  List[Any]				      =     pad_token_id
            a_     :  Dict				      =     decoder_start_token_id
            a_     :  Tuple				      =     use_cache
            a_     :  List[Any]				      =     max_position_embeddings

            a_     :  List[str]				      =     None
            a_     :  Any				      =     decoder_seq_length
            a_     :  List[Any]				      =     2
            a_     :  Optional[Any]				      =     1

       def     UpperCAmelCase( self   :  str     ):
            a_     :  Any				      =     ids_tensor([self.batch_size, self.decoder_seq_length]   ,				self.vocab_size     )

            a_     :  Any				      =     None
            if self.use_attention_mask:
                 a_     :  Optional[Any]				      =     ids_tensor([self.batch_size, self.decoder_seq_length]   ,				vocab_size=2     )

            a_     :  Tuple				      =     None
            if self.use_labels:
                 a_     :  int				      =     ids_tensor([self.batch_size, self.decoder_seq_length]   ,				self.vocab_size     )

            a_     :  int				      =     SpeechaTextaConfig(
                vocab_size=self.vocab_size   ,				d_model=self.d_model   ,				decoder_layers=self.decoder_layers   ,				decoder_ffn_dim=self.decoder_ffn_dim   ,				decoder_attention_heads=self.decoder_attention_heads   ,				eos_token_id=self.eos_token_id   ,				bos_token_id=self.bos_token_id   ,				use_cache=self.use_cache   ,				pad_token_id=self.pad_token_id   ,				decoder_start_token_id=self.decoder_start_token_id   ,				max_position_embeddings=self.max_position_embeddings   ,				)

            return (
                config,
                input_ids,
                attention_mask,
                lm_labels,
            )

       def     UpperCAmelCase( self   :  Tuple   ,				lowerCamelCase_   :  List[str]   ,				lowerCamelCase_   :  str   ,				lowerCamelCase_   :  Optional[int]   ,				lowerCamelCase_   :  List[str]   ,				):
            a_     :  List[Any]				      =     True
            a_     :  Optional[int]				      =     SpeechaTextaDecoder(config=lowerCamelCase_     ).to(lowerCamelCase_     ).eval()
            a_     :  Optional[int]				      =     input_ids[:2]

            input_ids[input_ids == 0] += 1
            # first forward pass
            a_     :  Union[str, Any]				      =     model(lowerCamelCase_   ,				use_cache=lowerCamelCase_     )
            a_     :  Tuple				      =     model(lowerCamelCase_     )
            a_     :  str				      =     model(lowerCamelCase_   ,				use_cache=lowerCamelCase_     )

            self.parent.assertTrue(len(lowerCamelCase_     ) == len(lowerCamelCase_     )     )
            self.parent.assertTrue(len(lowerCamelCase_     ) == len(lowerCamelCase_     ) + 1     )

            a_     :  Tuple				      =     outputs["""past_key_values"""]

            # create hypothetical next token and extent to next_input_ids
            a_     :  Any				      =     ids_tensor((2, 1)   ,				config.vocab_size - 1     ) + 1

            # append to next input_ids and
            a_     :  str				      =     torch.cat([input_ids, next_tokens]   ,				dim=-1     )

            print(lowerCamelCase_     )

            a_     :  Any				      =     model(lowerCamelCase_     )["""last_hidden_state"""]
            a_     :  int				      =     model(lowerCamelCase_   ,				past_key_values=lowerCamelCase_     )["""last_hidden_state"""]

            # select random slice
            a_     :  Tuple				      =     ids_tensor((1,)   ,				output_from_past.shape[-1]     ).item()
            a_     :  List[str]				      =     output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
            a_     :  List[Any]				      =     output_from_past[:, 0, random_slice_idx].detach()

            # test that outputs are equal for slice
            assert torch.allclose(lowerCamelCase_   ,				lowerCamelCase_   ,				atol=1E-3     )

       def     UpperCAmelCase( self   :  Any     ):
            a_     :  List[Any]				      =     self.prepare_config_and_inputs()
            (
                (
                a_
            )      ,      (
                a_
            )      ,      (
                a_
            )      ,      (
                a_
            )      ,      
            )     :  Optional[int]				      =     config_and_inputs

            a_     :  Tuple				      =     {
                """input_ids""": input_ids,
                """attention_mask""": attention_mask,
            }
            return config, inputs_dict


@require_torch
class 	a__						(    lowerCAmelCase_    ,							lowerCAmelCase_    ,							lowerCAmelCase_    ,							unittest.TestCase     ):
       lowerCamelCase__:  int				     =  (SpeechaTextaDecoder, SpeechaTextaForCausalLM) if is_torch_available() else ()
       lowerCamelCase__:  int				     =  (SpeechaTextaForCausalLM,) if is_torch_available() else ()
       lowerCamelCase__:  int				     =  {"""text-generation""": SpeechaTextaForCausalLM} if is_torch_available() else {}
       lowerCamelCase__:  Dict				     =  True
       lowerCamelCase__:  List[Any]				     =  False

       def     UpperCAmelCase( self   :  List[str]   ,				):
            a_     :  Optional[int]				      =     SpeechaTextaStandaloneDecoderModelTester(self   ,				is_training=lowerCamelCase_     )
            a_     :  str				      =     ConfigTester(self   ,				config_class=lowerCamelCase_     )

       def     UpperCAmelCase( self   :  Optional[Any]     ):
            pass

       def     UpperCAmelCase( self   :  Optional[Any]     ):
            pass

       def     UpperCAmelCase( self   :  Optional[Any]     ):
            pass

       def     UpperCAmelCase( self   :  str     ):
            self.config_tester.run_common_tests()

       def     UpperCAmelCase( self   :  Dict     ):
            a_     :  int				      =     self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_decoder_model_past(*lowerCamelCase_     )

       def     UpperCAmelCase( self   :  int     ):
            return







import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
       import torch

       from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
       import pyanvml.pyanvml as nvml


__lowerCamelCase   	=  logging.get_logger(__name__)


class 	a__						(    lowerCAmelCase_     ):
       lowerCamelCase__:  PyTorchBenchmarkArguments
       lowerCamelCase__:  PretrainedConfig
       lowerCamelCase__:  str				     =  "PyTorch"

       @property
       def     UpperCAmelCase( self   :  str     ):
            return torch.__version__

       def     UpperCAmelCase( self   :  str   ,				lowerCamelCase_   :  str   ,				lowerCamelCase_   :  int   ,				lowerCamelCase_   :  int     ):
            a_     :  Tuple				      =     self._prepare_inference_func(lowerCamelCase_   ,				lowerCamelCase_   ,				lowerCamelCase_     )
            return self._measure_speed(_inference     )

       def     UpperCAmelCase( self   :  Optional[Any]   ,				lowerCamelCase_   :  str   ,				lowerCamelCase_   :  int   ,				lowerCamelCase_   :  int     ):
            a_     :  List[str]				      =     self._prepare_inference_func(lowerCamelCase_   ,				lowerCamelCase_   ,				lowerCamelCase_     )
            return self._measure_memory(_inference     )

       def     UpperCAmelCase( self   :  List[str]   ,				lowerCamelCase_   :  str   ,				lowerCamelCase_   :  int   ,				lowerCamelCase_   :  int     ):
            a_     :  str				      =     self._prepare_train_func(lowerCamelCase_   ,				lowerCamelCase_   ,				lowerCamelCase_     )
            return self._measure_speed(_train     )

       def     UpperCAmelCase( self   :  str   ,				lowerCamelCase_   :  str   ,				lowerCamelCase_   :  int   ,				lowerCamelCase_   :  int     ):
            a_     :  List[Any]				      =     self._prepare_train_func(lowerCamelCase_   ,				lowerCamelCase_   ,				lowerCamelCase_     )
            return self._measure_memory(_train     )

       def     UpperCAmelCase( self   :  Union[str, Any]   ,				lowerCamelCase_   :  str   ,				lowerCamelCase_   :  int   ,				lowerCamelCase_   :  int     ):
            a_     :  Union[str, Any]				      =     self.config_dict[model_name]

            if self.args.torchscript:
                 a_     :  Optional[Any]				      =     True

            a_     :  Any				      =     (
                hasattr(lowerCamelCase_   ,				"""architectures"""     )
                and isinstance(config.architectures   ,				lowerCamelCase_     )
                and len(config.architectures     ) > 0
            )
            if not self.args.only_pretrain_model and has_model_class_in_config:
                 try:
                      a_     :  Union[str, Any]				      =     config.architectures[0]
                      a_     :  Any				      =     __import__("""transformers"""   ,				fromlist=[model_class]     )
                      a_     :  List[str]				      =     getattr(lowerCamelCase_   ,				lowerCamelCase_     )
                      a_     :  Dict				      =     model_cls(lowerCamelCase_     )
                 except ImportError:
                      raise ImportError(
                          F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                          """ set `--only_pretrain_model` or `args.only_pretrain_model=True`."""     )
            else:
                 a_     :  str				      =     MODEL_MAPPING[config.__class__](lowerCamelCase_     )

            model.eval()
            model.to(self.args.device     )

            # encoder-decoder has vocab size saved differently
            a_     :  Union[str, Any]				      =     config.vocab_size if hasattr(lowerCamelCase_   ,				"""vocab_size"""     ) else config.encoder.vocab_size
            a_     :  Dict				      =     torch.randint(lowerCamelCase_   ,				(batch_size, sequence_length)   ,				dtype=torch.long   ,				device=self.args.device     )

            if self.args.fpaa:
                 logger.info("""Running training in Mixed Precision..."""     )
                 if not self.args.is_gpu:
                      raise ValueError("""Mixed precision is possible only for GPU."""     )
                 # amp seems to have memory leaks so that memory usage
                 # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
                 model.half()

            if self.args.torchscript:
                 with torch.no_grad():
                      a_     :  Optional[int]				      =     torch.jit.trace(lowerCamelCase_   ,				lowerCamelCase_     )
            else:
                 a_     :  Optional[Any]				      =     model

            def encoder_decoder_forward():
                 with torch.no_grad():
                      a_     :  Any				      =     inference_model(lowerCamelCase_   ,				decoder_input_ids=lowerCamelCase_     )
                 return outputs

            def encoder_forward():
                 with torch.no_grad():
                      a_     :  Dict				      =     inference_model(lowerCamelCase_     )
                 return outputs

            a_     :  Union[str, Any]				      =     encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
            return _forward

       def     UpperCAmelCase( self   :  int   ,				lowerCamelCase_   :  str   ,				lowerCamelCase_   :  int   ,				lowerCamelCase_   :  int     ):
            a_     :  Optional[int]				      =     self.config_dict[model_name]

            a_     :  str				      =     (
                hasattr(lowerCamelCase_   ,				"""architectures"""     )
                and isinstance(config.architectures   ,				lowerCamelCase_     )
                and len(config.architectures     ) > 0
            )
            if not self.args.only_pretrain_model and has_model_class_in_config:
                 try:
                      a_     :  Optional[int]				      =     config.architectures[0]
                      a_     :  List[Any]				      =     __import__("""transformers"""   ,				fromlist=[model_class]     )
                      a_     :  List[str]				      =     getattr(lowerCamelCase_   ,				lowerCamelCase_     )
                      a_     :  Any				      =     model_cls(lowerCamelCase_     )
                 except ImportError:
                      raise ImportError(
                          F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                          """ set `--only_pretrain_model` or `args.only_pretrain_model=True`."""     )
            else:
                 a_     :  Union[str, Any]				      =     MODEL_WITH_LM_HEAD_MAPPING[config.__class__](lowerCamelCase_     )

            if self.args.torchscript:
                 raise NotImplementedError("""Training for torchscript is currently not implemented"""     )
            else:
                 a_     :  Dict				      =     model

            model.train()
            model.to(self.args.device     )

            # encoder-decoder has vocab size saved differently
            a_     :  Any				      =     config.vocab_size if hasattr(lowerCamelCase_   ,				"""vocab_size"""     ) else config.encoder.vocab_size
            a_     :  Tuple				      =     torch.randint(lowerCamelCase_   ,				(batch_size, sequence_length)   ,				dtype=torch.long   ,				device=self.args.device     )

            if self.args.fpaa:
                 logger.info("""Running training in Mixed Precision..."""     )
                 if not self.args.is_gpu:
                      raise ValueError("""Mixed precision is possible only for GPU."""     )

                 # amp seems to have memory leaks so that memory usage
                 # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
                 model.half()

            def compute_loss_and_backprob_encoder():
                 a_     :  Dict				      =     train_model(lowerCamelCase_   ,				labels=lowerCamelCase_     )[0]
                 loss.backward()
                 return loss

            def compute_loss_and_backprob_encoder_decoder():
                 a_     :  Optional[int]				      =     train_model(lowerCamelCase_   ,				decoder_input_ids=lowerCamelCase_   ,				labels=lowerCamelCase_     )[0]
                 loss.backward()
                 return loss

            a_     :  Optional[Any]				      =     (
                compute_loss_and_backprob_encoder_decoder
                if config.is_encoder_decoder
                else compute_loss_and_backprob_encoder
            )
            return _train

       def     UpperCAmelCase( self   :  str   ,				lowerCamelCase_   :  Optional[int]     ):
            try:
                 if self.args.is_tpu or self.args.torchscript:
                      # run additional 10 times to stabilize compilation for tpu and torchscript
                      logger.info("""Do inference on TPU or torchscript. Running model 5 times to stabilize compilation"""     )
                      timeit.repeat(
                          lowerCamelCase_   ,				repeat=1   ,				number=5   ,				)

                 # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
                 a_     :  List[str]				      =     timeit.repeat(
                     lowerCamelCase_   ,				repeat=self.args.repeat   ,				number=1_0   ,				)

                 if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
                      import torch_xla.debug.metrics as met

                      self.print_fn(met.metrics_report()     )

                 return min(lowerCamelCase_     ) / 1_0.0
            except RuntimeError as e:
                 self.print_fn(F'''Doesn\'t fit on GPU. {e}'''     )
                 return "N/A"

       def     UpperCAmelCase( self   :  Tuple   ,				lowerCamelCase_   :  Callable[[], None]     ):
            try:
                 if self.args.trace_memory_line_by_line:
                      a_     :  Dict				      =     start_memory_tracing("""transformers"""     )

                 if self.args.is_tpu:
                      # tpu
                      raise NotImplementedError(
                          """Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with"""
                          """ `--no-memory` or `args.memory=False`"""     )
                 elif self.args.is_gpu:
                      if not is_pyanvml_available():
                           logger.warning(
                               """py3nvml not installed, we won't log GPU memory usage. """
                               """Install py3nvml (pip install py3nvml) to log information about GPU."""     )
                           a_     :  str				      =     """N/A"""
                      else:
                           logger.info(
                               """Measuring total GPU usage on GPU device. Make sure to not have additional processes running"""
                               """ on the same GPU."""     )
                           # init nvml
                           nvml.nvmlInit()
                           func()
                           a_     :  List[Any]				      =     nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx     )
                           a_     :  Tuple				      =     nvml.nvmlDeviceGetMemoryInfo(lowerCamelCase_     )
                           a_     :  Any				      =     meminfo.used
                           a_     :  Any				      =     Memory(lowerCamelCase_     )
                           # shutdown nvml
                           nvml.nvmlShutdown()
                 else:
                      # cpu
                      a_     :  Union[str, Any]				      =     measure_peak_memory_cpu(lowerCamelCase_     )
                      a_     :  Optional[Any]				      =     Memory(lowerCamelCase_     ) if isinstance(lowerCamelCase_   ,				lowerCamelCase_     ) else memory_bytes

                 if self.args.trace_memory_line_by_line:
                      a_     :  Dict				      =     stop_memory_tracing(lowerCamelCase_     )
                 else:
                      a_     :  Any				      =     None

                 return memory, summary
            except RuntimeError as e:
                 self.print_fn(F'''Doesn\'t fit on GPU. {e}'''     )
                 return "N/A", None



