





import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
  from transformers import LlamaTokenizerFast
except ImportError as e:
  warnings.warn(e)
  warnings.warn(
      'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
  )
  UpperCAmelCase_     							= None
UpperCAmelCase_     							= {
    '7B': 1_1008,
    '13B': 1_3824,
    '30B': 1_7920,
    '65B': 2_2016,
    '70B': 2_8672,
}
UpperCAmelCase_     							= {
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}




def 		lowerCAmelCase_    (  __UpperCAmelCase:			str		,   __UpperCAmelCase:			str=1		,   __UpperCAmelCase:			Optional[Any]=256      )      ->	List[str]:
    return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3      )      ) + multiple_of - 1) // multiple_of)




def 		lowerCAmelCase_    (  __UpperCAmelCase:			Dict      )      ->	Dict:
    with open(__UpperCAmelCase		,   '''r'''      ) as f:
        return json.load(__UpperCAmelCase      )




def 		lowerCAmelCase_    (  __UpperCAmelCase:			List[str]		,   __UpperCAmelCase:			Any      )      ->	Optional[int]:
    with open(__UpperCAmelCase		,   '''w'''      ) as f:
        json.dump(__UpperCAmelCase		,   __UpperCAmelCase      )




def 		lowerCAmelCase_    (  __UpperCAmelCase:			Dict		,   __UpperCAmelCase:			List[Any]		,   __UpperCAmelCase:			Any		,   __UpperCAmelCase:			Dict=True      )      ->	int:
    os.makedirs(__UpperCAmelCase		,   exist_ok=__UpperCAmelCase      )
    UpperCamelCase__							:				Union[str, Any]          =      os.path.join(__UpperCAmelCase		,   '''tmp'''      )
    os.makedirs(__UpperCAmelCase		,   exist_ok=__UpperCAmelCase      )

    UpperCamelCase__							:				Optional[Any]          =      read_json(os.path.join(__UpperCAmelCase		,   '''params.json'''      )      )
    UpperCamelCase__							:				str          =      NUM_SHARDS[model_size]
    UpperCamelCase__							:				Any          =      params['''n_layers''']
    UpperCamelCase__							:				Union[str, Any]          =      params['''n_heads''']
    UpperCamelCase__							:				Dict          =      n_heads // num_shards
    UpperCamelCase__							:				Any          =      params['''dim''']
    UpperCamelCase__							:				List[Any]          =      dim // n_heads
    UpperCamelCase__							:				Any          =      10000.0
    UpperCamelCase__							:				Union[str, Any]          =      1.0 / (base ** (torch.arange(0		,   __UpperCAmelCase		,   2      ).float() / dims_per_head))

    if "n_kv_heads" in params:
        UpperCamelCase__							:				List[Any]          =      params['''n_kv_heads''']  # for GQA / MQA
        UpperCamelCase__							:				Optional[Any]          =      n_heads_per_shard // num_key_value_heads
        UpperCamelCase__							:				Union[str, Any]          =      dim // num_key_value_heads
    else:  # compatibility with other checkpoints
        UpperCamelCase__							:				Any          =      n_heads
        UpperCamelCase__							:				Optional[int]          =      n_heads_per_shard
        UpperCamelCase__							:				Any          =      dim

    # permute for sliced rotary
    def permute(__UpperCAmelCase:			List[str]		,   __UpperCAmelCase:			int=n_heads		,   __UpperCAmelCase:			List[Any]=dim		,   __UpperCAmelCase:			List[str]=dim      ):
        return w.view(__UpperCAmelCase		,   dima // n_heads // 2		,   2		,   __UpperCAmelCase      ).transpose(1		,   2      ).reshape(__UpperCAmelCase		,   __UpperCAmelCase      )

    print(f"Fetching all parameters from the checkpoint at {input_base_path}."      )
    # Load weights
    if model_size == "7B":
        # Not sharded
        # (The sharded implementation would also work, but this is simpler.)
        UpperCamelCase__							:				int          =      torch.load(os.path.join(__UpperCAmelCase		,   '''consolidated.00.pth'''      )		,   map_location='''cpu'''      )
    else:
        # Sharded
        UpperCamelCase__							:				Optional[Any]          =      [
            torch.load(os.path.join(__UpperCAmelCase		,   f"consolidated.{i:02d}.pth"      )		,   map_location='''cpu'''      )
            for i in range(__UpperCAmelCase      )
        ]
    UpperCamelCase__							:				Union[str, Any]          =      0
    UpperCamelCase__							:				Optional[int]          =      {'''weight_map''': {}}
    for layer_i in range(__UpperCAmelCase      ):
        UpperCamelCase__							:				List[Any]          =      f"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
        if model_size == "7B":
            # Unsharded
            UpperCamelCase__							:				int          =      {
                f"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
                    loaded[f"layers.{layer_i}.attention.wq.weight"]      ),
                f"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
                    loaded[f"layers.{layer_i}.attention.wk.weight"]      ),
                f"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[f"layers.{layer_i}.attention.wv.weight"],
                f"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[f"layers.{layer_i}.attention.wo.weight"],
                f"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w1.weight"],
                f"model.layers.{layer_i}.mlp.down_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w2.weight"],
                f"model.layers.{layer_i}.mlp.up_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w3.weight"],
                f"model.layers.{layer_i}.input_layernorm.weight": loaded[f"layers.{layer_i}.attention_norm.weight"],
                f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[f"layers.{layer_i}.ffn_norm.weight"],
            }
        else:
            # Sharded
            # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
            # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
            # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

            UpperCamelCase__							:				str          =      {
                f"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
                    f"layers.{layer_i}.attention_norm.weight"
                ].clone(),
                f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
                    f"layers.{layer_i}.ffn_norm.weight"
                ].clone(),
            }
            UpperCamelCase__							:				List[Any]          =      permute(
                torch.cat(
                    [
                        loaded[i][f"layers.{layer_i}.attention.wq.weight"].view(__UpperCAmelCase		,   __UpperCAmelCase		,   __UpperCAmelCase      )
                        for i in range(__UpperCAmelCase      )
                    ]		,   dim=0		,   ).reshape(__UpperCAmelCase		,   __UpperCAmelCase      )      )
            UpperCamelCase__							:				Dict          =      permute(
                torch.cat(
                    [
                        loaded[i][f"layers.{layer_i}.attention.wk.weight"].view(
                            __UpperCAmelCase		,   __UpperCAmelCase		,   __UpperCAmelCase      )
                        for i in range(__UpperCAmelCase      )
                    ]		,   dim=0		,   ).reshape(__UpperCAmelCase		,   __UpperCAmelCase      )		,   __UpperCAmelCase		,   __UpperCAmelCase		,   __UpperCAmelCase		,   )
            UpperCamelCase__							:				Any          =      torch.cat(
                [
                    loaded[i][f"layers.{layer_i}.attention.wv.weight"].view(
                        __UpperCAmelCase		,   __UpperCAmelCase		,   __UpperCAmelCase      )
                    for i in range(__UpperCAmelCase      )
                ]		,   dim=0		,   ).reshape(__UpperCAmelCase		,   __UpperCAmelCase      )

            UpperCamelCase__							:				int          =      torch.cat(
                [loaded[i][f"layers.{layer_i}.attention.wo.weight"] for i in range(__UpperCAmelCase      )]		,   dim=1      )
            UpperCamelCase__							:				Any          =      torch.cat(
                [loaded[i][f"layers.{layer_i}.feed_forward.w1.weight"] for i in range(__UpperCAmelCase      )]		,   dim=0      )
            UpperCamelCase__							:				str          =      torch.cat(
                [loaded[i][f"layers.{layer_i}.feed_forward.w2.weight"] for i in range(__UpperCAmelCase      )]		,   dim=1      )
            UpperCamelCase__							:				str          =      torch.cat(
                [loaded[i][f"layers.{layer_i}.feed_forward.w3.weight"] for i in range(__UpperCAmelCase      )]		,   dim=0      )

        UpperCamelCase__							:				Dict          =      inv_freq
        for k, v in state_dict.items():
            UpperCamelCase__							:				List[Any]          =      filename
            param_count += v.numel()
        torch.save(__UpperCAmelCase		,   os.path.join(__UpperCAmelCase		,   __UpperCAmelCase      )      )

    UpperCamelCase__							:				Dict          =      f"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
    if model_size == "7B":
        # Unsharded
        UpperCamelCase__							:				Optional[int]          =      {
            '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
            '''model.norm.weight''': loaded['''norm.weight'''],
            '''lm_head.weight''': loaded['''output.weight'''],
        }
    else:
        UpperCamelCase__							:				Union[str, Any]          =      {
            '''model.norm.weight''': loaded[0]['''norm.weight'''],
            '''model.embed_tokens.weight''': torch.cat(
                [loaded[i]['''tok_embeddings.weight'''] for i in range(__UpperCAmelCase      )]		,   dim=1      ),
            '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(__UpperCAmelCase      )]		,   dim=0      ),
        }

    for k, v in state_dict.items():
        UpperCamelCase__							:				Any          =      filename
        param_count += v.numel()
    torch.save(__UpperCAmelCase		,   os.path.join(__UpperCAmelCase		,   __UpperCAmelCase      )      )

    # Write configs
    UpperCamelCase__							:				int          =      {'''total_size''': param_count * 2}
    write_json(__UpperCAmelCase		,   os.path.join(__UpperCAmelCase		,   '''pytorch_model.bin.index.json'''      )      )
    UpperCamelCase__							:				str          =      params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
    UpperCamelCase__							:				Tuple          =      params['''multiple_of'''] if '''multiple_of''' in params else 256
    UpperCamelCase__							:				Dict          =      LlamaConfig(
        hidden_size=__UpperCAmelCase		,   intermediate_size=compute_intermediate_size(__UpperCAmelCase		,   __UpperCAmelCase		,   __UpperCAmelCase      )		,   num_attention_heads=params['''n_heads''']		,   num_hidden_layers=params['''n_layers''']		,   rms_norm_eps=params['''norm_eps''']		,   num_key_value_heads=__UpperCAmelCase		,   )
    config.save_pretrained(__UpperCAmelCase      )

    # Make space so we can load the model properly now.
    del state_dict
    del loaded
    gc.collect()

    print('''Loading the checkpoint in a Llama model.'''      )
    UpperCamelCase__							:				Any          =      LlamaForCausalLM.from_pretrained(__UpperCAmelCase		,   torch_dtype=torch.floataa		,   low_cpu_mem_usage=__UpperCAmelCase      )
    # Avoid saving this as part of the config.
    del model.config._name_or_path

    print('''Saving in the Transformers format.'''      )
    model.save_pretrained(__UpperCAmelCase		,   safe_serialization=__UpperCAmelCase      )
    shutil.rmtree(__UpperCAmelCase      )




def 		lowerCAmelCase_    (  __UpperCAmelCase:			List[str]		,   __UpperCAmelCase:			str      )      ->	List[Any]:
    # Initialize the tokenizer based on the `spm` model
    UpperCamelCase__							:				Tuple          =      LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
    print(f"Saving a {tokenizer_class.__name__} to {tokenizer_path}."      )
    UpperCamelCase__							:				List[Any]          =      tokenizer_class(__UpperCAmelCase      )
    tokenizer.save_pretrained(__UpperCAmelCase      )




def 		lowerCAmelCase_    (  )      ->	int:
    UpperCamelCase__							:				Dict          =      argparse.ArgumentParser()
    parser.add_argument(
        '''--input_dir'''		,   help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''		,   )
    parser.add_argument(
        '''--model_size'''		,   choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']		,   )
    parser.add_argument(
        '''--output_dir'''		,   help='''Location to write HF model and tokenizer'''		,   )
    parser.add_argument('''--safe_serialization'''		,   type=__UpperCAmelCase		,   help='''Whether or not to save using `safetensors`.'''      )
    UpperCamelCase__							:				Optional[Any]          =      parser.parse_args()
    if args.model_size != "tokenizer_only":
        write_model(
            model_path=args.output_dir		,   input_base_path=os.path.join(args.input_dir		,   args.model_size      )		,   model_size=args.model_size		,   safe_serialization=args.safe_serialization		,   )
    UpperCamelCase__							:				str          =      os.path.join(args.input_dir		,   '''tokenizer.model'''      )
    write_tokenizer(args.output_dir		,   __UpperCAmelCase      )


if __name__ == "__main__":
  main()








from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TFConvBertForMaskedLM,
      TFConvBertForMultipleChoice,
      TFConvBertForQuestionAnswering,
      TFConvBertForSequenceClassification,
      TFConvBertForTokenClassification,
      TFConvBertModel,
  )






class     lowercase__       :

   '''simple docstring'''


   def __init__(    self,		__magic_name__,		__magic_name__=13,		__magic_name__=7,		__magic_name__=True,		__magic_name__=True,		__magic_name__=True,		__magic_name__=True,		__magic_name__=99,		__magic_name__=32,		__magic_name__=2,		__magic_name__=4,		__magic_name__=37,		__magic_name__="gelu",		__magic_name__=0.1,		__magic_name__=0.1,		__magic_name__=512,		__magic_name__=16,		__magic_name__=2,		__magic_name__=0.02,		__magic_name__=3,		__magic_name__=4,		__magic_name__=None,		) ->      List[str]:






       """simple docstring"""
       UpperCamelCase__							:				List[Any]          =      parent
       UpperCamelCase__							:				int          =      13
       UpperCamelCase__							:				Optional[Any]          =      7
       UpperCamelCase__							:				Dict          =      True
       UpperCamelCase__							:				Dict          =      True
       UpperCamelCase__							:				str          =      True
       UpperCamelCase__							:				List[str]          =      True
       UpperCamelCase__							:				Tuple          =      99
       UpperCamelCase__							:				Any          =      384
       UpperCamelCase__							:				Tuple          =      2
       UpperCamelCase__							:				int          =      4
       UpperCamelCase__							:				Optional[int]          =      37
       UpperCamelCase__							:				str          =      '''gelu'''
       UpperCamelCase__							:				Any          =      0.1
       UpperCamelCase__							:				Dict          =      0.1
       UpperCamelCase__							:				Tuple          =      512
       UpperCamelCase__							:				int          =      16
       UpperCamelCase__							:				int          =      2
       UpperCamelCase__							:				Any          =      0.02
       UpperCamelCase__							:				Optional[Any]          =      3
       UpperCamelCase__							:				Any          =      4
       UpperCamelCase__							:				Any          =      128
       UpperCamelCase__							:				List[str]          =      2
       UpperCamelCase__							:				List[str]          =      9
       UpperCamelCase__							:				Optional[int]          =      1
       UpperCamelCase__							:				List[Any]          =      None




   def       UpperCamelCase__  (    self							) ->      Dict:






       """simple docstring"""
       UpperCamelCase__							:				Tuple          =      ids_tensor([self.batch_size, self.seq_length],		self.vocab_size							)

       UpperCamelCase__							:				Optional[int]          =      None
       if self.use_input_mask:
           UpperCamelCase__							:				Dict          =      random_attention_mask([self.batch_size, self.seq_length]							)

       UpperCamelCase__							:				Dict          =      None
       if self.use_token_type_ids:
           UpperCamelCase__							:				List[str]          =      ids_tensor([self.batch_size, self.seq_length],		self.type_vocab_size							)

       UpperCamelCase__							:				str          =      None
       UpperCamelCase__							:				Union[str, Any]          =      None
       UpperCamelCase__							:				Dict          =      None
       if self.use_labels:
           UpperCamelCase__							:				str          =      ids_tensor([self.batch_size],		self.type_sequence_label_size							)
           UpperCamelCase__							:				List[str]          =      ids_tensor([self.batch_size, self.seq_length],		self.num_labels							)
           UpperCamelCase__							:				Tuple          =      ids_tensor([self.batch_size],		self.num_choices							)

       UpperCamelCase__							:				Any          =      ConvBertConfig(
           vocab_size=self.vocab_size,		hidden_size=self.hidden_size,		num_hidden_layers=self.num_hidden_layers,		num_attention_heads=self.num_attention_heads,		intermediate_size=self.intermediate_size,		hidden_act=self.hidden_act,		hidden_dropout_prob=self.hidden_dropout_prob,		attention_probs_dropout_prob=self.attention_probs_dropout_prob,		max_position_embeddings=self.max_position_embeddings,		type_vocab_size=self.type_vocab_size,		initializer_range=self.initializer_range,		return_dict=__magic_name__,		)

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__							) ->      Tuple:






       """simple docstring"""
       UpperCamelCase__							:				Tuple          =      TFConvBertModel(config=__magic_name__							)
       UpperCamelCase__							:				str          =      {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

       UpperCamelCase__							:				Any          =      [input_ids, input_mask]
       UpperCamelCase__							:				int          =      model(__magic_name__							)

       UpperCamelCase__							:				int          =      model(__magic_name__							)

       self.parent.assertEqual(result.last_hidden_state.shape,		(self.batch_size, self.seq_length, self.hidden_size)							)




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__							) ->      int:






       """simple docstring"""
       UpperCamelCase__							:				Any          =      TFConvBertForMaskedLM(config=__magic_name__							)
       UpperCamelCase__							:				List[str]          =      {
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
           '''token_type_ids''': token_type_ids,
       }
       UpperCamelCase__							:				Tuple          =      model(__magic_name__							)
       self.parent.assertEqual(result.logits.shape,		(self.batch_size, self.seq_length, self.vocab_size)							)




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__							) ->      Dict:






       """simple docstring"""
       UpperCamelCase__							:				str          =      self.num_labels
       UpperCamelCase__							:				Optional[int]          =      TFConvBertForSequenceClassification(config=__magic_name__							)
       UpperCamelCase__							:				str          =      {
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
           '''token_type_ids''': token_type_ids,
       }

       UpperCamelCase__							:				Any          =      model(__magic_name__							)
       self.parent.assertEqual(result.logits.shape,		(self.batch_size, self.num_labels)							)




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__							) ->      Tuple:






       """simple docstring"""
       UpperCamelCase__							:				int          =      self.num_choices
       UpperCamelCase__							:				str          =      TFConvBertForMultipleChoice(config=__magic_name__							)
       UpperCamelCase__							:				Optional[int]          =      tf.tile(tf.expand_dims(__magic_name__,		1							),		(1, self.num_choices, 1)							)
       UpperCamelCase__							:				Any          =      tf.tile(tf.expand_dims(__magic_name__,		1							),		(1, self.num_choices, 1)							)
       UpperCamelCase__							:				Tuple          =      tf.tile(tf.expand_dims(__magic_name__,		1							),		(1, self.num_choices, 1)							)
       UpperCamelCase__							:				int          =      {
           '''input_ids''': multiple_choice_inputs_ids,
           '''attention_mask''': multiple_choice_input_mask,
           '''token_type_ids''': multiple_choice_token_type_ids,
       }
       UpperCamelCase__							:				Tuple          =      model(__magic_name__							)
       self.parent.assertEqual(result.logits.shape,		(self.batch_size, self.num_choices)							)




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__							) ->      Optional[int]:






       """simple docstring"""
       UpperCamelCase__							:				Optional[Any]          =      self.num_labels
       UpperCamelCase__							:				Any          =      TFConvBertForTokenClassification(config=__magic_name__							)
       UpperCamelCase__							:				str          =      {
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
           '''token_type_ids''': token_type_ids,
       }
       UpperCamelCase__							:				Dict          =      model(__magic_name__							)
       self.parent.assertEqual(result.logits.shape,		(self.batch_size, self.seq_length, self.num_labels)							)




   def       UpperCamelCase__  (    self,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__,		__magic_name__							) ->      str:






       """simple docstring"""
       UpperCamelCase__							:				Union[str, Any]          =      TFConvBertForQuestionAnswering(config=__magic_name__							)
       UpperCamelCase__							:				Optional[Any]          =      {
           '''input_ids''': input_ids,
           '''attention_mask''': input_mask,
           '''token_type_ids''': token_type_ids,
       }

       UpperCamelCase__							:				List[str]          =      model(__magic_name__							)
       self.parent.assertEqual(result.start_logits.shape,		(self.batch_size, self.seq_length)							)
       self.parent.assertEqual(result.end_logits.shape,		(self.batch_size, self.seq_length)							)




   def       UpperCamelCase__  (    self							) ->      str:






       """simple docstring"""
       UpperCamelCase__							:				Any          =      self.prepare_config_and_inputs()
       (
           (
           UpperCamelCase__
       )   ,(
           UpperCamelCase__
       )   ,(
           UpperCamelCase__
       )   ,(
           UpperCamelCase__
       )   ,(
           UpperCamelCase__
       )   ,(
           UpperCamelCase__
       )   ,(
           UpperCamelCase__
       )   ,
       )							:				Any          =      config_and_inputs
       UpperCamelCase__							:				List[str]          =      {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
       return config, inputs_dict






@require_tf
class     lowercase__       (	__lowerCamelCase			,				__lowerCamelCase			,				unittest.TestCase				):

   '''simple docstring'''


   a						:							Union[str, Any]   		=		(
       (
           TFConvBertModel,
           TFConvBertForMaskedLM,
           TFConvBertForQuestionAnswering,
           TFConvBertForSequenceClassification,
           TFConvBertForTokenClassification,
           TFConvBertForMultipleChoice,
       )
       if is_tf_available()
       else ()
   )
   a						:							int   		=		(
       {
           "feature-extraction": TFConvBertModel,
           "fill-mask": TFConvBertForMaskedLM,
           "question-answering": TFConvBertForQuestionAnswering,
           "text-classification": TFConvBertForSequenceClassification,
           "token-classification": TFConvBertForTokenClassification,
           "zero-shot": TFConvBertForSequenceClassification,
       }
       if is_tf_available()
       else {}
   )
   a						:							Optional[Any]   		=		False
   a						:							List[str]   		=		False
   a						:							List[Any]   		=		False




   def       UpperCamelCase__  (    self							) ->      Optional[Any]:






       """simple docstring"""
       UpperCamelCase__							:				List[str]          =      TFConvBertModelTester(self							)
       UpperCamelCase__							:				Optional[int]          =      ConfigTester(self,		config_class=__magic_name__,		hidden_size=37							)




   def       UpperCamelCase__  (    self							) ->      int:






       """simple docstring"""
       self.config_tester.run_common_tests()




   def       UpperCamelCase__  (    self							) ->      List[Any]:






       """simple docstring"""
       UpperCamelCase__							:				Any          =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*__magic_name__							)




   def       UpperCamelCase__  (    self							) ->      Tuple:






       """simple docstring"""
       UpperCamelCase__							:				str          =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*__magic_name__							)




   def       UpperCamelCase__  (    self							) ->      Dict:






       """simple docstring"""
       UpperCamelCase__							:				List[Any]          =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*__magic_name__							)




   def       UpperCamelCase__  (    self							) ->      Optional[Any]:






       """simple docstring"""
       UpperCamelCase__							:				str          =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*__magic_name__							)




   def       UpperCamelCase__  (    self							) ->      Tuple:






       """simple docstring"""
       UpperCamelCase__							:				Optional[int]          =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*__magic_name__							)




   def       UpperCamelCase__  (    self							) ->      Dict:






       """simple docstring"""
       UpperCamelCase__							:				str          =      self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*__magic_name__							)




   @slow
   def       UpperCamelCase__  (    self							) ->      Optional[int]:






       """simple docstring"""
       UpperCamelCase__   ,UpperCamelCase__							:				Tuple          =      self.model_tester.prepare_config_and_inputs_for_common()
       UpperCamelCase__							:				Union[str, Any]          =      True
       UpperCamelCase__							:				Optional[int]          =      True

       if hasattr(__magic_name__,		'''use_cache'''							):
           UpperCamelCase__							:				Union[str, Any]          =      True

       UpperCamelCase__							:				Tuple          =      getattr(self.model_tester,		'''encoder_seq_length''',		self.model_tester.seq_length							)
       UpperCamelCase__							:				Any          =      getattr(self.model_tester,		'''key_length''',		__magic_name__							)

       for model_class in self.all_model_classes:
           UpperCamelCase__							:				Optional[int]          =      self._prepare_for_class(__magic_name__,		__magic_name__							)
           UpperCamelCase__							:				Any          =      model_class(__magic_name__							)
           UpperCamelCase__							:				List[str]          =      len(model(__magic_name__							)							)

           with tempfile.TemporaryDirectory() as tmpdirname:
               model.save_pretrained(__magic_name__,		saved_model=__magic_name__							)
               UpperCamelCase__							:				List[Any]          =      os.path.join(__magic_name__,		'''saved_model''',		'''1'''							)
               UpperCamelCase__							:				Dict          =      tf.keras.models.load_model(__magic_name__							)
               UpperCamelCase__							:				Optional[int]          =      model(__magic_name__							)

               if self.is_encoder_decoder:
                   UpperCamelCase__							:				Union[str, Any]          =      outputs['''encoder_hidden_states''']
                   UpperCamelCase__							:				Any          =      outputs['''encoder_attentions''']
               else:
                   UpperCamelCase__							:				Optional[Any]          =      outputs['''hidden_states''']
                   UpperCamelCase__							:				Dict          =      outputs['''attentions''']

               self.assertEqual(len(__magic_name__							),		__magic_name__							)

               UpperCamelCase__							:				Optional[Any]          =      getattr(
                   self.model_tester,		'''expected_num_hidden_layers''',		self.model_tester.num_hidden_layers + 1							)

               self.assertEqual(len(__magic_name__							),		__magic_name__							)
               self.assertListEqual(
                   list(output_hidden_states[0].shape[-2:]							),		[self.model_tester.seq_length, self.model_tester.hidden_size],		)

               self.assertEqual(len(__magic_name__							),		self.model_tester.num_hidden_layers							)
               self.assertListEqual(
                   list(output_attentions[0].shape[-3:]							),		[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length],		)




   @slow
   def       UpperCamelCase__  (    self							) ->      str:






       """simple docstring"""
       UpperCamelCase__							:				str          =      TFConvBertModel.from_pretrained('''YituTech/conv-bert-base'''							)
       self.assertIsNotNone(__magic_name__							)




   def       UpperCamelCase__  (    self							) ->      Dict:






       """simple docstring"""
       UpperCamelCase__   ,UpperCamelCase__							:				int          =      self.model_tester.prepare_config_and_inputs_for_common()
       UpperCamelCase__							:				str          =      True
       UpperCamelCase__							:				Union[str, Any]          =      getattr(self.model_tester,		'''decoder_seq_length''',		self.model_tester.seq_length							)
       UpperCamelCase__							:				str          =      getattr(self.model_tester,		'''encoder_seq_length''',		self.model_tester.seq_length							)
       UpperCamelCase__							:				List[str]          =      getattr(self.model_tester,		'''key_length''',		__magic_name__							)
       UpperCamelCase__							:				int          =      getattr(self.model_tester,		'''key_length''',		__magic_name__							)

       def check_decoder_attentions_output(__magic_name__							):
           UpperCamelCase__							:				List[str]          =      len(__magic_name__							)
           self.assertEqual(out_len % 2,		0							)
           UpperCamelCase__							:				Optional[Any]          =      outputs.decoder_attentions
           self.assertEqual(len(__magic_name__							),		self.model_tester.num_hidden_layers							)
           self.assertListEqual(
               list(decoder_attentions[0].shape[-3:]							),		[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length],		)

       def check_encoder_attentions_output(__magic_name__							):
           UpperCamelCase__							:				int          =      [
               t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
           ]
           self.assertEqual(len(__magic_name__							),		self.model_tester.num_hidden_layers							)
           self.assertListEqual(
               list(attentions[0].shape[-3:]							),		[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length],		)

       for model_class in self.all_model_classes:
           UpperCamelCase__							:				Dict          =      True
           UpperCamelCase__							:				str          =      False
           UpperCamelCase__							:				Optional[Any]          =      model_class(__magic_name__							)
           UpperCamelCase__							:				Dict          =      model(self._prepare_for_class(__magic_name__,		__magic_name__							)							)
           UpperCamelCase__							:				int          =      len(__magic_name__							)
           self.assertEqual(config.output_hidden_states,		__magic_name__							)
           check_encoder_attentions_output(__magic_name__							)

           if self.is_encoder_decoder:
               UpperCamelCase__							:				str          =      model_class(__magic_name__							)
               UpperCamelCase__							:				List[str]          =      model(self._prepare_for_class(__magic_name__,		__magic_name__							)							)
               self.assertEqual(config.output_hidden_states,		__magic_name__							)
               check_decoder_attentions_output(__magic_name__							)

           # Check that output attentions can also be changed via the config
           del inputs_dict["output_attentions"]
           UpperCamelCase__							:				List[Any]          =      True
           UpperCamelCase__							:				Dict          =      model_class(__magic_name__							)
           UpperCamelCase__							:				Optional[Any]          =      model(self._prepare_for_class(__magic_name__,		__magic_name__							)							)
           self.assertEqual(config.output_hidden_states,		__magic_name__							)
           check_encoder_attentions_output(__magic_name__							)

           # Check attention is always last and order is fine
           UpperCamelCase__							:				List[Any]          =      True
           UpperCamelCase__							:				List[Any]          =      True
           UpperCamelCase__							:				Dict          =      model_class(__magic_name__							)
           UpperCamelCase__							:				List[Any]          =      model(self._prepare_for_class(__magic_name__,		__magic_name__							)							)

           self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1),		len(__magic_name__							)							)
           self.assertEqual(model.config.output_hidden_states,		__magic_name__							)
           check_encoder_attentions_output(__magic_name__							)






@require_tf
class     lowercase__       (	unittest.TestCase				):

   '''simple docstring'''


   @slow
   def       UpperCamelCase__  (    self							) ->      int:






       """simple docstring"""
       UpperCamelCase__							:				Union[str, Any]          =      TFConvBertModel.from_pretrained('''YituTech/conv-bert-base'''							)
       UpperCamelCase__							:				Tuple          =      tf.constant([[0, 1, 2, 3, 4, 5]]							)
       UpperCamelCase__							:				int          =      model(__magic_name__							)[0]

       UpperCamelCase__							:				Dict          =      [1, 6, 768]
       self.assertEqual(output.shape,		__magic_name__							)

       UpperCamelCase__							:				Union[str, Any]          =      tf.constant(
           [
               [
                   [-0.0347_5493, -0.468_6034, -0.3063_8832],
                   [0.2263_7248, -0.2698_8646, -0.742_3424],
                   [0.1032_4868, -0.4501_3508, -0.5828_0784],
               ]
           ]							)
       tf.debugging.assert_near(output[:, :3, :3],		__magic_name__,		atol=1E-4							)

