code: 
  
  import collections
  
  from .utils import ExplicitEnum, is_torch_available, logging
  
  
  if is_torch_available():
     import torch
  
  
  A_		:			Union[str, Any]										=   logging.get_logger(__name__)
  class 					A_   :
  
  
      '''simple docstring'''
  
  
  
  
  
  
      def __init__(self     ,      lowercase__     ,      lowercase__=21     ,      lowercase__=[]     ,      lowercase__=None  )     ->      Tuple:
          __UpperCAmelCase									=			model
          __UpperCAmelCase									=			trace_batch_nums
          __UpperCAmelCase									=			abort_after_batch_num
  
          # keep a LIFO buffer of frames to dump as soon as inf/nan is encountered to give context to the problem emergence
          __UpperCAmelCase									=			collections.deque([]     ,      __UpperCamelCase  )
          __UpperCAmelCase									=			[]
          __UpperCAmelCase									=			0
          __UpperCAmelCase									=			0
          __UpperCAmelCase									=			False
          __UpperCAmelCase									=			'''                 '''
  
          self.analyse_model()
  
          self.register_forward_hook()
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__=None  )     ->      Optional[int]:
          if frame is not None:
              self.expand_frame(__UpperCamelCase  )
          self.frames.append('''\n'''.join(self.frame  )  )
          __UpperCAmelCase									=			[]  # start a new frame
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      Dict:
          self.frame.append(__UpperCamelCase  )
  
  
      def      lowerCAmelCase_						(self  )     ->      List[Any]:
          print('''\n'''.join(self.frames  )  )
          __UpperCAmelCase									=			[]
  
  
      def      lowerCAmelCase_						(self  )     ->      str:
          __UpperCAmelCase									=			[]
  
  
      def      lowerCAmelCase_						(self  )     ->      Any:
          print(F'''\nDetected inf/nan during batch_number={self.batch_number}'''  )
          print(F'''Last {len(self.frames  )} forward frames:'''  )
          print(F'''{'abs min':8} {'abs max':8} metadata'''  )
          print('''\n'''.join(self.frames  )  )
          print('''\n\n'''  )
          __UpperCAmelCase									=			[]
  
  
      def      lowerCAmelCase_						(self  )     ->      Dict:
          # extract the fully qualified module names, to be able to report at run time. e.g.:
          # encoder.block.2.layer.0.SelfAttention.o
          #
          # for shared weights only the first shared module name will be registered
          __UpperCAmelCase									=			{m: name for name, m in self.model.named_modules()}
          # self.longest_module_name = max(len(v) for v in self.module_names.values())
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__  )     ->      Tuple:
          if torch.is_tensor(__UpperCamelCase  ):
              self.expand_frame(get_abs_min_max(__UpperCamelCase     ,      __UpperCamelCase  )  )
              if detect_overflow(__UpperCamelCase     ,      __UpperCamelCase  ):
                  __UpperCAmelCase									=			True
          elif var is None:
              self.expand_frame(F'''{'None':>17} {ctx}'''  )
          else:
              self.expand_frame(F'''{'not a tensor':>17} {ctx}'''  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[int]:
          self.expand_frame(F'''\n\n{self.prefix} *** Starting batch number={self.batch_number} ***'''  )
          self.expand_frame(F'''{'abs min':8} {'abs max':8} metadata'''  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[Any]:
          self.expand_frame(F'''{self.prefix} *** Finished batch number={self.batch_number-1} ***\n\n'''  )
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__  )     ->      Dict:
          self.expand_frame(F'''{self.prefix} {self.module_names[module]} {module.__class__.__name__}'''  )
  
          # params
          for name, p in module.named_parameters(recurse=__UpperCamelCase  ):
              self.analyse_variable(__UpperCamelCase     ,      __UpperCamelCase  )
  
          # inputs
          if isinstance(__UpperCamelCase     ,      __UpperCamelCase  ):
              for i, x in enumerate(__UpperCamelCase  ):
                  self.analyse_variable(__UpperCamelCase     ,      F'''input[{i}]'''  )
          else:
              self.analyse_variable(__UpperCamelCase     ,      '''input'''  )
  
          # outputs
          if isinstance(__UpperCamelCase     ,      __UpperCamelCase  ):
              for i, x in enumerate(__UpperCamelCase  ):
                  # possibly a tuple of tuples
                  if isinstance(__UpperCamelCase     ,      __UpperCamelCase  ):
                      for j, y in enumerate(__UpperCamelCase  ):
                          self.analyse_variable(__UpperCamelCase     ,      F'''output[{i}][{j}]'''  )
                  else:
                      self.analyse_variable(__UpperCamelCase     ,      F'''output[{i}]'''  )
          else:
              self.analyse_variable(__UpperCamelCase     ,      '''output'''  )
  
          self.save_frame()
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[int]:
          self.model.apply(self._register_forward_hook  )
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__  )     ->      Optional[int]:
          module.register_forward_hook(self.forward_hook  )
  
  
  
  
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__  )     ->      Any:
          # - input is a tuple of packed inputs (could be non-Tensors)
          # - output could be a Tensor or a tuple of Tensors and non-Tensors
  
          __UpperCAmelCase									=			False
  
          __UpperCAmelCase									=			True if self.batch_number in self.trace_batch_nums else False
          if trace_mode:
              self.reset_saved_frames()
  
          if self.total_calls == 0:
              self.batch_start_frame()
          self.total_calls += 1
  
          # count batch numbers - the very first forward hook of the batch will be called when the
          # batch completes - i.e. it gets called very last - we know this batch has finished
          if module == self.model:
              self.batch_number += 1
              __UpperCAmelCase									=			True
  
          self.create_frame(__UpperCamelCase     ,      __UpperCamelCase     ,      __UpperCamelCase  )
  
          # if last_frame_of_batch:
          #     self.batch_end_frame()
  
          if trace_mode:
              self.trace_frames()
  
          if last_frame_of_batch:
              self.batch_start_frame()
  
          if self.detected_overflow and not trace_mode:
              self.dump_saved_frames()
  
              # now we can abort, as it's pointless to continue running
              raise ValueError(
                  '''DebugUnderflowOverflow: inf/nan detected, aborting as there is no point running further. '''
                  '''Please scroll up above this traceback to see the activation values prior to this event.'''  )
  
          # abort after certain batch if requested to do so
          if self.abort_after_batch_num is not None and self.batch_number > self.abort_after_batch_num:
              raise ValueError(
                  F'''DebugUnderflowOverflow: aborting after {self.batch_number} batches due to'''
                  F''' `abort_after_batch_num={self.abort_after_batch_num}` arg'''  )
  
  
  
  
  
  
  def       __a       (   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE      )      ->     Tuple:
  
  
  
  
  
  
      '''simple docstring'''
      __UpperCAmelCase									=			var.abs()
      return f'''{abs_var.min():8.2e} {abs_var.max():8.2e} {ctx}'''
  
  
  def       __a       (   SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE      )      ->     Dict:
  
  
  
  
  
  
      '''simple docstring'''
      __UpperCAmelCase									=			False
      if torch.isnan(lowercase_      ).any().item():
          __UpperCAmelCase									=			True
          print(f'''{ctx} has nans'''      )
      if torch.isinf(lowercase_      ).any().item():
          __UpperCAmelCase									=			True
          print(f'''{ctx} has infs'''      )
  
      # if needed to monitor large elements can enable the following
      if 0:  # and detected:
          __UpperCAmelCase									=			var[torch.ge(var.abs()       ,   1_0_0      )]
          if naaa.numel() > 0:
              print(f'''{ctx}:  n100={naaa.numel()}'''      )
          __UpperCAmelCase									=			var[torch.ge(var.abs()       ,   1_0_0_0      )]
          if naaaa.numel() > 0:
              print(f'''{ctx}: n1000={naaaa.numel()}'''      )
          __UpperCAmelCase									=			var[torch.ge(var.abs()       ,   1_0_0_0_0      )]
          if naaaaa.numel() > 0:
              print(f'''{ctx}: n10000={naaaaa.numel()}'''      )
  
      if 0:
          print(f'''min={var.min():9.2e} max={var.max():9.2e}'''      )
  
      if 0:
          print(f'''min={var.min():9.2e} max={var.max():9.2e} var={var.var():9.2e} mean={var.mean():9.2e} ({ctx})'''      )
  
      return detected
  class 					A_   (      UpperCAmelCase_						):
  
  
      '''simple docstring'''
  
  
  
  
  
  
      a__     							=       "underflow_overflow"
      a__     							=       "tpu_metrics_debug"
  
  
  
  
  
style_context: 
  
  from __future__ import annotations
  
  import unittest
  
  from transformers import is_tf_available
  from transformers.testing_utils import require_tf, slow
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_tf_available():
         import tensorflow as tf
  
         from transformers import (
             TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
             TFXLMForMultipleChoice,
             TFXLMForQuestionAnsweringSimple,
             TFXLMForSequenceClassification,
             TFXLMForTokenClassification,
             TFXLMModel,
             TFXLMWithLMHeadModel,
             XLMConfig,
         )
  class 					A_   :
  
  
      '''simple docstring'''
  
  
  
  
  
  
      def __init__(self     ,      lowercase__     ,      )     ->      Tuple:
          __UpperCAmelCase									=			parent
          __UpperCAmelCase									=			13
          __UpperCAmelCase									=			7
          __UpperCAmelCase									=			True
          __UpperCAmelCase									=			True
          __UpperCAmelCase									=			True
          __UpperCAmelCase									=			True
          __UpperCAmelCase									=			True
          __UpperCAmelCase									=			False
          __UpperCAmelCase									=			False
          __UpperCAmelCase									=			False
          __UpperCAmelCase									=			2
          __UpperCAmelCase									=			99
          __UpperCAmelCase									=			0
          __UpperCAmelCase									=			32
          __UpperCAmelCase									=			2
          __UpperCAmelCase									=			4
          __UpperCAmelCase									=			0.1
          __UpperCAmelCase									=			0.1
          __UpperCAmelCase									=			512
          __UpperCAmelCase									=			16
          __UpperCAmelCase									=			2
          __UpperCAmelCase									=			0.02
          __UpperCAmelCase									=			3
          __UpperCAmelCase									=			4
          __UpperCAmelCase									=			'''last'''
          __UpperCAmelCase									=			True
          __UpperCAmelCase									=			None
          __UpperCAmelCase									=			0
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[Any]:
          __UpperCAmelCase									=			ids_tensor([self.batch_size, self.seq_length]     ,      self.vocab_size  )
          __UpperCAmelCase									=			random_attention_mask([self.batch_size, self.seq_length]     ,      dtype=tf.floataa  )
  
          __UpperCAmelCase									=			None
          if self.use_input_lengths:
              __UpperCAmelCase									=			(
                  ids_tensor([self.batch_size]     ,      vocab_size=2  ) + self.seq_length - 2
              )  # small variation of seq_length
  
          __UpperCAmelCase									=			None
          if self.use_token_type_ids:
              __UpperCAmelCase									=			ids_tensor([self.batch_size, self.seq_length]     ,      self.n_langs  )
  
          __UpperCAmelCase									=			None
          __UpperCAmelCase									=			None
          __UpperCAmelCase									=			None
          if self.use_labels:
              __UpperCAmelCase									=			ids_tensor([self.batch_size]     ,      self.type_sequence_label_size  )
              __UpperCAmelCase									=			ids_tensor([self.batch_size, self.seq_length]     ,      self.num_labels  )
              __UpperCAmelCase									=			ids_tensor([self.batch_size]     ,      2     ,      dtype=tf.floataa  )
              __UpperCAmelCase									=			ids_tensor([self.batch_size]     ,      self.num_choices  )
  
          __UpperCAmelCase									=			XLMConfig(
              vocab_size=self.vocab_size     ,      n_special=self.n_special     ,      emb_dim=self.hidden_size     ,      n_layers=self.num_hidden_layers     ,      n_heads=self.num_attention_heads     ,      dropout=self.hidden_dropout_prob     ,      attention_dropout=self.attention_probs_dropout_prob     ,      gelu_activation=self.gelu_activation     ,      sinusoidal_embeddings=self.sinusoidal_embeddings     ,      asm=self.asm     ,      causal=self.causal     ,      n_langs=self.n_langs     ,      max_position_embeddings=self.max_position_embeddings     ,      initializer_range=self.initializer_range     ,      summary_type=self.summary_type     ,      use_proj=self.use_proj     ,      bos_token_id=self.bos_token_id     ,      )
  
          return (
              config,
              input_ids,
              token_type_ids,
              input_lengths,
              sequence_labels,
              token_labels,
              is_impossible_labels,
              choice_labels,
              input_mask,
          )
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      )     ->      List[Any]:
          __UpperCAmelCase									=			TFXLMModel(config=lowercase__  )
          __UpperCAmelCase									=			{'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
          __UpperCAmelCase									=			model(lowercase__  )
  
          __UpperCAmelCase									=			[input_ids, input_mask]
          __UpperCAmelCase									=			model(lowercase__  )
          self.parent.assertEqual(result.last_hidden_state.shape     ,      (self.batch_size, self.seq_length, self.hidden_size)  )
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      )     ->      List[str]:
          __UpperCAmelCase									=			TFXLMWithLMHeadModel(lowercase__  )
  
          __UpperCAmelCase									=			{'''input_ids''': input_ids, '''lengths''': input_lengths, '''langs''': token_type_ids}
          __UpperCAmelCase									=			model(lowercase__  )
  
          __UpperCAmelCase									=			outputs
  
          self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.vocab_size)  )
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      )     ->      Optional[Any]:
          __UpperCAmelCase									=			TFXLMForQuestionAnsweringSimple(lowercase__  )
  
          __UpperCAmelCase									=			{'''input_ids''': input_ids, '''lengths''': input_lengths}
  
          __UpperCAmelCase									=			model(lowercase__  )
  
          self.parent.assertEqual(result.start_logits.shape     ,      (self.batch_size, self.seq_length)  )
          self.parent.assertEqual(result.end_logits.shape     ,      (self.batch_size, self.seq_length)  )
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      )     ->      Dict:
          __UpperCAmelCase									=			TFXLMForSequenceClassification(lowercase__  )
  
          __UpperCAmelCase									=			{'''input_ids''': input_ids, '''lengths''': input_lengths}
  
          __UpperCAmelCase									=			model(lowercase__  )
  
          self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.type_sequence_label_size)  )
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      )     ->      int:
          __UpperCAmelCase									=			self.num_labels
          __UpperCAmelCase									=			TFXLMForTokenClassification(config=lowercase__  )
          __UpperCAmelCase									=			{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}
          __UpperCAmelCase									=			model(lowercase__  )
          self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.seq_length, self.num_labels)  )
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      )     ->      Tuple:
          __UpperCAmelCase									=			self.num_choices
          __UpperCAmelCase									=			TFXLMForMultipleChoice(config=lowercase__  )
          __UpperCAmelCase									=			tf.tile(tf.expand_dims(lowercase__     ,      1  )     ,      (1, self.num_choices, 1)  )
          __UpperCAmelCase									=			tf.tile(tf.expand_dims(lowercase__     ,      1  )     ,      (1, self.num_choices, 1)  )
          __UpperCAmelCase									=			tf.tile(tf.expand_dims(lowercase__     ,      1  )     ,      (1, self.num_choices, 1)  )
          __UpperCAmelCase									=			{
              '''input_ids''': multiple_choice_inputs_ids,
              '''attention_mask''': multiple_choice_input_mask,
              '''token_type_ids''': multiple_choice_token_type_ids,
          }
          __UpperCAmelCase									=			model(lowercase__  )
          self.parent.assertEqual(result.logits.shape     ,      (self.batch_size, self.num_choices)  )
  
  
  
  
  
  
      def      lowerCAmelCase_						(self  )     ->      str:
          __UpperCAmelCase									=			self.prepare_config_and_inputs()
          (
              (
              __UpperCAmelCase
          )					,     (
              __UpperCAmelCase
          )					,     (
              __UpperCAmelCase
          )					,     (
              __UpperCAmelCase
          )					,     (
              __UpperCAmelCase
          )					,     (
              __UpperCAmelCase
          )					,     (
              __UpperCAmelCase
          )					,     (
              __UpperCAmelCase
          )					,     (
              __UpperCAmelCase
          )					,     
          )									=			config_and_inputs
          __UpperCAmelCase									=			{
              '''input_ids''': input_ids,
              '''token_type_ids''': token_type_ids,
              '''langs''': token_type_ids,
              '''lengths''': input_lengths,
          }
          return config, inputs_dict
  
  
  
  
  
  
  @require_tf
  class 					A_   (      _a ,     _a ,     unittest.TestCase						):
  
  
      '''simple docstring'''
  
  
  
  
  
  
      a__     							=       (
          (
              TFXLMModel,
              TFXLMWithLMHeadModel,
              TFXLMForSequenceClassification,
              TFXLMForQuestionAnsweringSimple,
              TFXLMForTokenClassification,
              TFXLMForMultipleChoice,
          )
          if is_tf_available()
          else ()
      )
      a__     							=       (
          (TFXLMWithLMHeadModel,) if is_tf_available() else ()
      )  # TODO (PVP): Check other models whether language generation is also applicable
      a__     							=       (
          {
              "feature-extraction": TFXLMModel,
              "fill-mask": TFXLMWithLMHeadModel,
              "question-answering": TFXLMForQuestionAnsweringSimple,
              "text-classification": TFXLMForSequenceClassification,
              "text-generation": TFXLMWithLMHeadModel,
              "token-classification": TFXLMForTokenClassification,
              "zero-shot": TFXLMForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )
      a__     							=       False
      a__     							=       False
  
  
      def      lowerCAmelCase_						(self     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__     ,      lowercase__  )     ->      Tuple:
          if (
              pipeline_test_casse_name == "QAPipelineTests"
              and tokenizer_name is not None
              and not tokenizer_name.endswith('''Fast'''  )
          ):
              # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
              # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
              # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
              return True
  
          return False
  
  
      def      lowerCAmelCase_						(self  )     ->      Tuple:
          __UpperCAmelCase									=			TFXLMModelTester(self  )
          __UpperCAmelCase									=			ConfigTester(self     ,      config_class=lowercase__     ,      emb_dim=37  )
  
  
      def      lowerCAmelCase_						(self  )     ->      str:
          self.config_tester.run_common_tests()
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[int]:
          __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_xlm_model(*lowercase__  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Any:
          __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_xlm_lm_head(*lowercase__  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Dict:
          __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_xlm_qa(*lowercase__  )
  
  
      def      lowerCAmelCase_						(self  )     ->      int:
          __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_xlm_sequence_classif(*lowercase__  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Optional[Any]:
          __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_xlm_for_token_classification(*lowercase__  )
  
  
      def      lowerCAmelCase_						(self  )     ->      Tuple:
          __UpperCAmelCase									=			self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_xlm_for_multiple_choice(*lowercase__  )
  
  
  
  
  
  
      @slow
      def      lowerCAmelCase_						(self  )     ->      int:
          for model_name in TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              __UpperCAmelCase									=			TFXLMModel.from_pretrained(lowercase__  )
              self.assertIsNotNone(lowercase__  )
  
  
  
  
  
  
  @require_tf
  class 					A_   (      unittest.TestCase						):
  
  
      '''simple docstring'''
  
  
  
  
  
  
      @slow
      def      lowerCAmelCase_						(self  )     ->      Tuple:
          __UpperCAmelCase									=			TFXLMWithLMHeadModel.from_pretrained('''xlm-mlm-en-2048'''  )
          __UpperCAmelCase									=			tf.convert_to_tensor([[14, 447]]     ,      dtype=tf.intaa  )  # the president
          __UpperCAmelCase									=			[
              14,
              447,
              14,
              447,
              14,
              447,
              14,
              447,
              14,
              447,
              14,
              447,
              14,
              447,
              14,
              447,
              14,
              447,
              14,
              447,
          ]  # the president the president the president the president the president the president the president the president the president the president
          # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
          __UpperCAmelCase									=			model.generate(lowercase__     ,      do_sample=lowercase__  )
          self.assertListEqual(output_ids[0].numpy().tolist()     ,      lowercase__  )
  
  
  
  
  