




import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis_float32 (there's also the fix_lavis branch)
# also note: to convert Vicuna checkpoints, we had to include /home/niels/python_projects/checkpoints/FastChat/vicuna-7b in lavis/configs/models/blip2/blip2_instruct_vicuna7b.yaml
# same for Vicuna-13b
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipImageProcessor,
    InstructBlipConfig,
    InstructBlipForConditionalGeneration,
    InstructBlipProcessor,
    InstructBlipQFormerConfig,
    InstructBlipVisionConfig,
    LlamaConfig,
    LlamaTokenizerFast,
    TaConfig,
    TaTokenizerFast,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD

def 			_SCREAMING_SNAKE_CASE			(							):


      '''simple docstring'''



      lowerCamelCase_		     =    'https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg'
      lowerCamelCase_		     =    Image.open(requests.get(lowercase	,							stream=lowercase     ).raw     ).convert('RGB'     )

      return image

def 			_SCREAMING_SNAKE_CASE			(							lowercase	:       Tuple     ):


      '''simple docstring'''



      lowerCamelCase_		     =    []
      # fmt: off

      # vision encoder
      rename_keys.append(('visual_encoder.cls_token', 'vision_model.embeddings.class_embedding')     )
      rename_keys.append(('visual_encoder.pos_embed', 'vision_model.embeddings.position_embedding')     )
      rename_keys.append(('visual_encoder.patch_embed.proj.weight', 'vision_model.embeddings.patch_embedding.weight')     )
      rename_keys.append(('visual_encoder.patch_embed.proj.bias', 'vision_model.embeddings.patch_embedding.bias')     )
      rename_keys.append(('ln_vision.weight', 'vision_model.post_layernorm.weight')     )
      rename_keys.append(('ln_vision.bias', 'vision_model.post_layernorm.bias')     )

      for i in range(config.vision_config.num_hidden_layers     ):
            rename_keys.append((f"""visual_encoder.blocks.{i}.norm1.weight""", f"""vision_model.encoder.layers.{i}.layer_norm1.weight""")     )
            rename_keys.append((f"""visual_encoder.blocks.{i}.norm1.bias""", f"""vision_model.encoder.layers.{i}.layer_norm1.bias""")     )
            rename_keys.append((f"""visual_encoder.blocks.{i}.norm2.weight""", f"""vision_model.encoder.layers.{i}.layer_norm2.weight""")     )
            rename_keys.append((f"""visual_encoder.blocks.{i}.norm2.bias""", f"""vision_model.encoder.layers.{i}.layer_norm2.bias""")     )
            rename_keys.append((f"""visual_encoder.blocks.{i}.attn.qkv.weight""", f"""vision_model.encoder.layers.{i}.self_attn.qkv.weight""")     )
            rename_keys.append((f"""visual_encoder.blocks.{i}.attn.proj.weight""", f"""vision_model.encoder.layers.{i}.self_attn.projection.weight""",)     )
            rename_keys.append((f"""visual_encoder.blocks.{i}.attn.proj.bias""", f"""vision_model.encoder.layers.{i}.self_attn.projection.bias""")     )
            rename_keys.append((f"""visual_encoder.blocks.{i}.mlp.fc1.weight""", f"""vision_model.encoder.layers.{i}.mlp.fc1.weight""")     )
            rename_keys.append((f"""visual_encoder.blocks.{i}.mlp.fc1.bias""", f"""vision_model.encoder.layers.{i}.mlp.fc1.bias""")     )
            rename_keys.append((f"""visual_encoder.blocks.{i}.mlp.fc2.weight""", f"""vision_model.encoder.layers.{i}.mlp.fc2.weight""")     )
            rename_keys.append((f"""visual_encoder.blocks.{i}.mlp.fc2.bias""", f"""vision_model.encoder.layers.{i}.mlp.fc2.bias""")     )

      # QFormer
      rename_keys.append(('Qformer.bert.embeddings.LayerNorm.weight', 'qformer.embeddings.layernorm.weight')     )
      rename_keys.append(('Qformer.bert.embeddings.LayerNorm.bias', 'qformer.embeddings.layernorm.bias')     )

      # fmt: on
      return rename_keys

def 			_SCREAMING_SNAKE_CASE			(							lowercase	:       int	,							lowercase	:       Dict	,							lowercase	:       List[str]     ):


      '''simple docstring'''



      lowerCamelCase_		     =    dct.pop(lowercase     )
      lowerCamelCase_		     =    val

def 			_SCREAMING_SNAKE_CASE			(							lowercase	:       Optional[Any]	,							lowercase	:       Tuple     ):


      '''simple docstring'''



      for i in range(config.vision_config.num_hidden_layers     ):
            # read in original q and v biases
            lowerCamelCase_		     =    state_dict.pop(f"""visual_encoder.blocks.{i}.attn.q_bias"""     )
            lowerCamelCase_		     =    state_dict.pop(f"""visual_encoder.blocks.{i}.attn.v_bias"""     )

            # next, set bias in the state dict
            lowerCamelCase_		     =    torch.cat((q_bias, torch.zeros_like(lowercase	,							requires_grad=lowercase     ), v_bias)     )
            lowerCamelCase_		     =    qkv_bias

def 			_SCREAMING_SNAKE_CASE			(							lowercase	:       Any     ):


      '''simple docstring'''



      lowerCamelCase_		     =    3_64 if 'coco' in model_name else 2_24
      lowerCamelCase_		     =    InstructBlipVisionConfig(image_size=lowercase     ).to_dict()

      # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
      # seems like flan-T5 models don't have bos_token_id properly set?
      if "t5-xl" in model_name:
            lowerCamelCase_		     =    TaConfig.from_pretrained('google/flan-t5-xl'	,							dense_act_fn='gelu'	,							bos_token_id=1     ).to_dict()
      elif "t5-xxl" in model_name:
            lowerCamelCase_		     =    TaConfig.from_pretrained('google/flan-t5-xxl'	,							dense_act_fn='gelu'	,							bos_token_id=1     ).to_dict()
      elif "vicuna-7b" in model_name:
            lowerCamelCase_		     =    LlamaConfig.from_pretrained('decapoda-research/llama-7b-hf'	,							vocab_size=3_20_01     ).to_dict()
      elif "vicuna-13b" in model_name:
            lowerCamelCase_		     =    LlamaConfig.from_pretrained('decapoda-research/llama-13b-hf'	,							vocab_size=3_20_01     ).to_dict()
      else:
            raise ValueError('Model name not supported'     )

      # the authors add one special "[DEC]" token to the vocab of Q-Former, hence vocab size = 30522 + 1
      lowerCamelCase_		     =    InstructBlipQFormerConfig(vocab_size=3_05_23     ).to_dict()
      lowerCamelCase_		     =    InstructBlipConfig(vision_config=lowercase	,							text_config=lowercase	,							qformer_config=lowercase     )

      return config, image_size

@torch.no_grad()
def 			_SCREAMING_SNAKE_CASE			(							lowercase	:       List[str]	,							lowercase	:       Optional[int]=None	,							lowercase	:       Optional[Any]=False     ):


      '''simple docstring'''



      lowerCamelCase_		     =    AutoTokenizer.from_pretrained('bert-base-uncased'	,							truncation_side='left'     )
      qformer_tokenizer.add_special_tokens({'bos_token': '[DEC]'}     )

      if "t5" in model_name:
            lowerCamelCase_		     =    TaTokenizerFast.from_pretrained('google/flan-t5-xl'	,							truncation_side='left'     )
      elif "vicuna" in model_name:
            # the following was used in the original implementation:
            # tokenizer = LlamaTokenizer.from_pretrained("huggyllama/llama-7b", use_fast=False, truncation_side="left")
            # tokenizer.add_special_tokens({"pad_token": "[PAD]"})
            # tokenizer.add_special_tokens({"bos_token": "</s>"})
            # tokenizer.add_special_tokens({"eos_token": "</s>"})
            # tokenizer.add_special_tokens({"unk_token": "</s>"})
            lowerCamelCase_		     =    LlamaTokenizerFast.from_pretrained(
                'huggyllama/llama-7b'	,							truncation_side='left'	,							bos_token='</s>'	,							unk_token='</s>'     )
            tokenizer.add_special_tokens({'pad_token': '[PAD]'}     )

      lowerCamelCase_     ,				lowerCamelCase_		     =    get_blipa_config(lowercase     )
      lowerCamelCase_		     =    InstructBlipForConditionalGeneration(lowercase     ).eval()

      lowerCamelCase_		     =    {
          'instructblip-vicuna-7b': ('blip2_vicuna_instruct', 'vicuna7b'),
          'instructblip-vicuna-13b': ('blip2_vicuna_instruct', 'vicuna13b'),
          'instructblip-flan-t5-xl': ('blip2_t5_instruct', 'flant5xl'),
          'instructblip-flan-t5-xxl': ('blip2_t5_instruct', 'flant5xxl'),
      }

      lowerCamelCase_     ,				lowerCamelCase_		     =    model_name_to_original[model_name]

      # load original model
      print('Loading original model...'     )
      lowerCamelCase_		     =    'cuda:1' if torch.cuda.is_available() else 'cpu'
      lowerCamelCase_		     =    'cuda:2' if torch.cuda.is_available() else 'cpu'
      lowerCamelCase_     ,				lowerCamelCase_     ,				lowerCamelCase_		     =    load_model_and_preprocess(
          name=lowercase	,							model_type=lowercase	,							is_eval=lowercase	,							device=lowercase     )
      original_model.eval()
      print('Done!'     )

      # update state dict keys
      lowerCamelCase_		     =    original_model.state_dict()
      lowerCamelCase_		     =    create_rename_keys(lowercase     )
      for src, dest in rename_keys:
            rename_key(lowercase	,							lowercase	,							lowercase     )

      # some keys can be renamed efficiently
      for key, val in state_dict.copy().items():
            lowerCamelCase_		     =    state_dict.pop(lowercase     )
            if key.startswith('Qformer.bert'     ):
                  lowerCamelCase_		     =    key.replace('Qformer.bert'	,							'qformer'     )
            if "attention.self" in key:
                  lowerCamelCase_		     =    key.replace('self'	,							'attention'     )
            if "llm_proj" in key:
                  lowerCamelCase_		     =    key.replace('llm_proj'	,							'language_projection'     )
            if "t5_proj" in key:
                  lowerCamelCase_		     =    key.replace('t5_proj'	,							'language_projection'     )
            if key.startswith('llm_model'     ):
                  lowerCamelCase_		     =    key.replace('llm_model'	,							'language_model'     )
            if key.startswith('t5'     ):
                  lowerCamelCase_		     =    key.replace('t5'	,							'language'     )
            lowerCamelCase_		     =    val

      # read in qv biases
      read_in_q_v_bias(lowercase	,							lowercase     )

      # note: weights get loaded in torch.float32 by default
      hf_model.load_state_dict(lowercase	,							strict=lowercase     )

      lowerCamelCase_		     =    load_demo_image()
      lowerCamelCase_		     =    'What is unusual about this image?'

      # create processor
      lowerCamelCase_		     =    BlipImageProcessor(
          size={'height': image_size, 'width': image_size}	,							image_mean=lowercase	,							image_std=lowercase     )
      lowerCamelCase_		     =    InstructBlipProcessor(
          image_processor=lowercase	,							tokenizer=lowercase	,							qformer_tokenizer=lowercase	,							)
      lowerCamelCase_		     =    processor(images=lowercase	,							text=lowercase	,							return_tensors='pt'     ).to(lowercase     )

      # make sure processor creates exact same pixel values
      lowerCamelCase_		     =    vis_processors['eval'](lowercase     ).unsqueeze(0     ).to(lowercase     )
      lowerCamelCase_		     =    inputs.pixel_values
      assert torch.allclose(original_pixel_values.to(pixel_values.device     )	,							lowercase     )

      original_model.to(lowercase     )
      hf_model.to(lowercase     )
      with torch.no_grad():
            if "vicuna" in model_name:
                  lowerCamelCase_		     =    original_model({'image': original_pixel_values, 'text_input': [prompt]}     ).logits
                  lowerCamelCase_		     =    hf_model(**lowercase     ).logits
            else:
                  lowerCamelCase_		     =    original_model(
                      {'image': original_pixel_values, 'text_input': [prompt], 'text_output': ['\n']}     ).logits
                  lowerCamelCase_		     =    tokenizer('\n'	,							return_tensors='pt'     ).input_ids.to(lowercase     )
                  lowerCamelCase_		     =    label_input_ids.masked_fill(label_input_ids == tokenizer.pad_token_id	,							-1_00     )
                  lowerCamelCase_		     =    hf_model(**lowercase	,							labels=lowercase     ).logits

      print('First values of original logits:'	,							original_logits[0, :3, :3]     )
      print('First values of HF logits:'	,							logits[0, :3, :3]     )

      # assert values
      assert original_logits.shape == logits.shape
      lowerCamelCase_		     =    1e-4 if 'vicuna' in model_name else 1e-5
      assert torch.allclose(original_logits.to(logits.device     )	,							lowercase	,							atol=lowercase     )
      print('Looks ok!'     )

      print('Generating with original model...'     )
      lowerCamelCase_		     =    original_model.generate({'image': original_pixel_values, 'prompt': prompt}	,							num_beams=5     )

      # important: we need to cast the weights of the HF model to the appropriate type
      print('Generating with HF model...'     )
      lowerCamelCase_		     =    hf_model.generate(
          **lowercase	,							do_sample=lowercase	,							num_beams=5	,							max_length=2_56	,							min_length=1	,							top_p=0.9	,							repetition_penalty=1.5	,							length_penalty=1.0	,							temperature=1	,							)
      if "vicuna" in model_name:
            # convert output id 0 to 2 (eos_token_id)
            # TODO add this in the generate method?
            lowerCamelCase_		     =    2
      print('Original generation:'	,							lowercase     )
      lowerCamelCase_		     =    processor.batch_decode(lowercase	,							skip_special_tokens=lowercase     )
      lowerCamelCase_		     =    [text.strip() for text in output_text]
      print('HF generation:'	,							lowercase     )

      if pytorch_dump_folder_path is not None:
            processor.save_pretrained(lowercase     )
            hf_model.save_pretrained(lowercase     )

      if push_to_hub:
            processor.push_to_hub(f"""Salesforce/{model_name}"""     )
            hf_model.push_to_hub(f"""Salesforce/{model_name}"""     )


if __name__ == "__main__":
 lowerCamelCase				:		Any     =   argparse.ArgumentParser()
 lowerCamelCase				:		Tuple     =   [
     "instructblip-vicuna-7b",
     "instructblip-vicuna-13b",
     "instructblip-flan-t5-xl",
     "instructblip-flan-t5-xxl",
 ]
 parser.add_argument(
     "--model_name",
     default="instructblip-flan-t5-xl",
     choices=choices,
     type=str,
     help="Path to hf config.json of model to convert",
 )
 parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.")
 parser.add_argument(
     "--push_to_hub",
     action="store_true",
     help="Whether to push the model and processor to the hub after converting",
 )

 lowerCamelCase				:		Tuple     =   parser.parse_args()

 convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)









from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import numpy as np
 import tensorflow as tf

 from transformers import (
     TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
     FlaubertConfig,
     TFFlaubertForMultipleChoice,
     TFFlaubertForQuestionAnsweringSimple,
     TFFlaubertForSequenceClassification,
     TFFlaubertForTokenClassification,
     TFFlaubertModel,
     TFFlaubertWithLMHeadModel,
 )







class   A:



 '''simple docstring'''


 def __init__(				self					:  str						,							A_					:  Optional[Any]						,							)   ->	str:

       """simple docstring"""






       lowerCamelCase_		     =    parent
       lowerCamelCase_		     =    13
       lowerCamelCase_		     =    7
       lowerCamelCase_		     =    True
       lowerCamelCase_		     =    True
       lowerCamelCase_		     =    True
       lowerCamelCase_		     =    True
       lowerCamelCase_		     =    True
       lowerCamelCase_		     =    False
       lowerCamelCase_		     =    False
       lowerCamelCase_		     =    False
       lowerCamelCase_		     =    2
       lowerCamelCase_		     =    99
       lowerCamelCase_		     =    0
       lowerCamelCase_		     =    32
       lowerCamelCase_		     =    2
       lowerCamelCase_		     =    4
       lowerCamelCase_		     =    0.1
       lowerCamelCase_		     =    0.1
       lowerCamelCase_		     =    512
       lowerCamelCase_		     =    16
       lowerCamelCase_		     =    2
       lowerCamelCase_		     =    0.02
       lowerCamelCase_		     =    3
       lowerCamelCase_		     =    4
       lowerCamelCase_		     =    'last'
       lowerCamelCase_		     =    True
       lowerCamelCase_		     =    None
       lowerCamelCase_		     =    0

 def 				a__ (				self					:  Dict  )   ->	Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_		     =    ids_tensor([self.batch_size, self.seq_length]						,							self.vocab_size  )
       lowerCamelCase_		     =    random_attention_mask([self.batch_size, self.seq_length]						,							dtype=tf.floataa  )

       lowerCamelCase_		     =    None
       if self.use_input_lengths:
             lowerCamelCase_		     =    (
                 ids_tensor([self.batch_size]						,							vocab_size=2  ) + self.seq_length - 2
             )  # small variation of seq_length

       lowerCamelCase_		     =    None
       if self.use_token_type_ids:
             lowerCamelCase_		     =    ids_tensor([self.batch_size, self.seq_length]						,							self.n_langs  )

       lowerCamelCase_		     =    None
       lowerCamelCase_		     =    None
       lowerCamelCase_		     =    None
       if self.use_labels:
             lowerCamelCase_		     =    ids_tensor([self.batch_size]						,							self.type_sequence_label_size  )
             lowerCamelCase_		     =    ids_tensor([self.batch_size, self.seq_length]						,							self.num_labels  )
             lowerCamelCase_		     =    ids_tensor([self.batch_size]						,							2						,							dtype=tf.floataa  )
             lowerCamelCase_		     =    ids_tensor([self.batch_size]						,							self.num_choices  )

       lowerCamelCase_		     =    FlaubertConfig(
           vocab_size=self.vocab_size						,							n_special=self.n_special						,							emb_dim=self.hidden_size						,							n_layers=self.num_hidden_layers						,							n_heads=self.num_attention_heads						,							dropout=self.hidden_dropout_prob						,							attention_dropout=self.attention_probs_dropout_prob						,							gelu_activation=self.gelu_activation						,							sinusoidal_embeddings=self.sinusoidal_embeddings						,							asm=self.asm						,							causal=self.causal						,							n_langs=self.n_langs						,							max_position_embeddings=self.max_position_embeddings						,							initializer_range=self.initializer_range						,							summary_type=self.summary_type						,							use_proj=self.use_proj						,							bos_token_id=self.bos_token_id						,							)

       return (
           config,
           input_ids,
           token_type_ids,
           input_lengths,
           sequence_labels,
           token_labels,
           is_impossible_labels,
           choice_labels,
           input_mask,
       )

 def 				a__ (				self					:  int						,							A_					:  List[str]						,							A_					:  List[Any]						,							A_					:  str						,							A_					:  List[Any]						,							A_					:  int						,							A_					:  Tuple						,							A_					:  Optional[int]						,							A_					:  Optional[int]						,							A_					:  str						,							)   ->	Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_		     =    TFFlaubertModel(config=A_  )
       lowerCamelCase_		     =    {'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
       lowerCamelCase_		     =    model(A_  )

       lowerCamelCase_		     =    [input_ids, input_mask]
       lowerCamelCase_		     =    model(A_  )
       self.parent.assertEqual(result.last_hidden_state.shape						,							(self.batch_size, self.seq_length, self.hidden_size)  )

 def 				a__ (				self					:  Tuple						,							A_					:  List[str]						,							A_					:  int						,							A_					:  List[Any]						,							A_					:  Any						,							A_					:  Any						,							A_					:  Dict						,							A_					:  str						,							A_					:  List[Any]						,							A_					:  Union[str, Any]						,							)   ->	List[str]:

       """simple docstring"""






       lowerCamelCase_		     =    TFFlaubertWithLMHeadModel(A_  )

       lowerCamelCase_		     =    {'input_ids': input_ids, 'lengths': input_lengths, 'langs': token_type_ids}
       lowerCamelCase_		     =    model(A_  )

       self.parent.assertEqual(result.logits.shape						,							(self.batch_size, self.seq_length, self.vocab_size)  )

 def 				a__ (				self					:  str						,							A_					:  Tuple						,							A_					:  Any						,							A_					:  Any						,							A_					:  List[Any]						,							A_					:  Dict						,							A_					:  List[Any]						,							A_					:  Union[str, Any]						,							A_					:  Optional[int]						,							A_					:  List[Any]						,							)   ->	Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_		     =    TFFlaubertForQuestionAnsweringSimple(A_  )

       lowerCamelCase_		     =    {'input_ids': input_ids, 'lengths': input_lengths}

       lowerCamelCase_		     =    model(A_  )

       self.parent.assertEqual(result.start_logits.shape						,							(self.batch_size, self.seq_length)  )
       self.parent.assertEqual(result.end_logits.shape						,							(self.batch_size, self.seq_length)  )

 def 				a__ (				self					:  Optional[int]						,							A_					:  List[Any]						,							A_					:  str						,							A_					:  List[str]						,							A_					:  Dict						,							A_					:  Optional[Any]						,							A_					:  Tuple						,							A_					:  str						,							A_					:  Optional[int]						,							A_					:  Tuple						,							)   ->	List[Any]:

       """simple docstring"""






       lowerCamelCase_		     =    TFFlaubertForSequenceClassification(A_  )

       lowerCamelCase_		     =    {'input_ids': input_ids, 'lengths': input_lengths}

       lowerCamelCase_		     =    model(A_  )

       self.parent.assertEqual(result.logits.shape						,							(self.batch_size, self.type_sequence_label_size)  )

 def 				a__ (				self					:  Dict						,							A_					:  Optional[Any]						,							A_					:  List[Any]						,							A_					:  int						,							A_					:  Any						,							A_					:  Union[str, Any]						,							A_					:  str						,							A_					:  Any						,							A_					:  Union[str, Any]						,							A_					:  List[str]						,							)   ->	Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_		     =    self.num_labels
       lowerCamelCase_		     =    TFFlaubertForTokenClassification(config=A_  )
       lowerCamelCase_		     =    {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
       lowerCamelCase_		     =    model(A_  )
       self.parent.assertEqual(result.logits.shape						,							(self.batch_size, self.seq_length, self.num_labels)  )

 def 				a__ (				self					:  List[Any]						,							A_					:  Optional[int]						,							A_					:  List[Any]						,							A_					:  Optional[int]						,							A_					:  Tuple						,							A_					:  Union[str, Any]						,							A_					:  int						,							A_					:  str						,							A_					:  Tuple						,							A_					:  str						,							)   ->	Optional[int]:

       """simple docstring"""






       lowerCamelCase_		     =    self.num_choices
       lowerCamelCase_		     =    TFFlaubertForMultipleChoice(config=A_  )
       lowerCamelCase_		     =    tf.tile(tf.expand_dims(A_						,							1  )						,							(1, self.num_choices, 1)  )
       lowerCamelCase_		     =    tf.tile(tf.expand_dims(A_						,							1  )						,							(1, self.num_choices, 1)  )
       lowerCamelCase_		     =    tf.tile(tf.expand_dims(A_						,							1  )						,							(1, self.num_choices, 1)  )
       lowerCamelCase_		     =    {
           'input_ids': multiple_choice_inputs_ids,
           'attention_mask': multiple_choice_input_mask,
           'token_type_ids': multiple_choice_token_type_ids,
       }
       lowerCamelCase_		     =    model(A_  )
       self.parent.assertEqual(result.logits.shape						,							(self.batch_size, self.num_choices)  )





 def 				a__ (				self					:  Union[str, Any]  )   ->	List[Any]:

       """simple docstring"""






       lowerCamelCase_		     =    self.prepare_config_and_inputs()
       (
           (
           lowerCamelCase_
       )     ,				(
           lowerCamelCase_
       )     ,				(
           lowerCamelCase_
       )     ,				(
           lowerCamelCase_
       )     ,				(
           lowerCamelCase_
       )     ,				(
           lowerCamelCase_
       )     ,				(
           lowerCamelCase_
       )     ,				(
           lowerCamelCase_
       )     ,				(
           lowerCamelCase_
       )     ,				
       )		     =    config_and_inputs
       lowerCamelCase_		     =    {
           'input_ids': input_ids,
           'token_type_ids': token_type_ids,
           'langs': token_type_ids,
           'lengths': input_lengths,
       }
       return config, inputs_dict







@require_tf
class   A(						UpperCamelCase							,      UpperCamelCase							,      unittest.TestCase   ):



 '''simple docstring'''


 UpperCamelCase						   =						(
     (
         TFFlaubertModel,
         TFFlaubertWithLMHeadModel,
         TFFlaubertForSequenceClassification,
         TFFlaubertForQuestionAnsweringSimple,
         TFFlaubertForTokenClassification,
         TFFlaubertForMultipleChoice,
     )
     if is_tf_available()
     else ()
 )
 UpperCamelCase						   =						(
     (TFFlaubertWithLMHeadModel,) if is_tf_available() else ()
 )  # TODO (PVP): Check other models whether language generation is also applicable
 UpperCamelCase						   =						(
     {
         '''feature-extraction''': TFFlaubertModel,
         '''fill-mask''': TFFlaubertWithLMHeadModel,
         '''question-answering''': TFFlaubertForQuestionAnsweringSimple,
         '''text-classification''': TFFlaubertForSequenceClassification,
         '''token-classification''': TFFlaubertForTokenClassification,
         '''zero-shot''': TFFlaubertForSequenceClassification,
     }
     if is_tf_available()
     else {}
 )
 UpperCamelCase						   =						False
 UpperCamelCase						   =						False

 def 				a__ (				self					:  Union[str, Any]						,							A_					:  Any						,							A_					:  List[Any]						,							A_					:  Union[str, Any]						,							A_					:  str						,							A_					:  List[str]  )   ->	Optional[Any]:

       """simple docstring"""






       if (
           pipeline_test_casse_name == "QAPipelineTests"
           and tokenizer_name is not None
           and not tokenizer_name.endswith('Fast'  )
       ):
             # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
             # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
             # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
             return True

       return False

 def 				a__ (				self					:  List[str]  )   ->	Optional[int]:

       """simple docstring"""






       lowerCamelCase_		     =    TFFlaubertModelTester(self  )
       lowerCamelCase_		     =    ConfigTester(self						,							config_class=A_						,							emb_dim=37  )

 def 				a__ (				self					:  List[str]  )   ->	str:

       """simple docstring"""






       self.config_tester.run_common_tests()

 def 				a__ (				self					:  int  )   ->	Optional[int]:

       """simple docstring"""






       lowerCamelCase_		     =    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_flaubert_model(*A_  )

 def 				a__ (				self					:  List[str]  )   ->	Any:

       """simple docstring"""






       lowerCamelCase_		     =    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_flaubert_lm_head(*A_  )

 def 				a__ (				self					:  Dict  )   ->	Tuple:

       """simple docstring"""






       lowerCamelCase_		     =    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_flaubert_qa(*A_  )

 def 				a__ (				self					:  Union[str, Any]  )   ->	Tuple:

       """simple docstring"""






       lowerCamelCase_		     =    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_flaubert_sequence_classif(*A_  )

 def 				a__ (				self					:  List[Any]  )   ->	Dict:

       """simple docstring"""






       lowerCamelCase_		     =    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_flaubert_for_token_classification(*A_  )

 def 				a__ (				self					:  int  )   ->	Optional[int]:

       """simple docstring"""






       lowerCamelCase_		     =    self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_flaubert_for_multiple_choice(*A_  )





 @slow
 def 				a__ (				self					:  Union[str, Any]  )   ->	Optional[int]:

       """simple docstring"""






       for model_name in TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             lowerCamelCase_		     =    TFFlaubertModel.from_pretrained(A_  )
             self.assertIsNotNone(A_  )







@require_tf
@require_sentencepiece
@require_tokenizers
class   A(						unittest.TestCase   ):



 '''simple docstring'''


 @slow
 def 				a__ (				self					:  Dict  )   ->	str:

       """simple docstring"""






       lowerCamelCase_		     =    TFFlaubertModel.from_pretrained('jplu/tf-flaubert-small-cased'  )

       lowerCamelCase_		     =    tf.convert_to_tensor(
           [[0, 158, 735, 2592, 1424, 6727, 82, 1]]						,							dtype=tf.intaa						,							)  # "J'aime flaubert !"

       lowerCamelCase_		     =    model(A_  )[0]
       lowerCamelCase_		     =    tf.TensorShape((1, 8, 512)  )
       self.assertEqual(output.shape						,							A_  )
       # compare the actual values for a slice.
       lowerCamelCase_		     =    tf.convert_to_tensor(
           [
               [
                   [-1.8768773, -1.566555, 0.27072418],
                   [-1.6920038, -0.5873505, 1.9329599],
                   [-2.9563985, -1.6993835, 1.7972052],
               ]
           ]						,							dtype=tf.floataa						,							)

       self.assertTrue(np.allclose(output[:, :3, :3].numpy()						,							expected_slice.numpy()						,							atol=1E-4  )  )



