
import argparse

import gdown
import numpy as np
import torch
from huggingface_hub import hf_hub_download

from transformers import (
    CLIPTokenizer,
    CLIPTokenizerFast,
    VideoMAEImageProcessor,
    XCLIPConfig,
    XCLIPModel,
    XCLIPProcessor,
    XCLIPTextConfig,
    XCLIPVisionConfig,
)
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		int:
			lowerCAmelCase__      :      int         =		XCLIPTextConfig()

			# derive patch size from model name
			lowerCAmelCase__      :      Any         =		model_name.find('patch'     )
			lowerCAmelCase__      :      Any         =		int(model_name[start_idx + len('patch'     ) : start_idx + len('patch'     ) + 2]     )
			lowerCAmelCase__      :      Any         =		XCLIPVisionConfig(patch_size=SCREAMING_SNAKE_CASE_       ,							num_frames=SCREAMING_SNAKE_CASE_     )

			if "large" in model_name:
						lowerCAmelCase__      :      List[str]         =		768
						lowerCAmelCase__      :      Optional[int]         =		3_072
						lowerCAmelCase__      :      Any         =		12

						lowerCAmelCase__      :      Optional[int]         =		1_024
						lowerCAmelCase__      :      List[Any]         =		4_096
						lowerCAmelCase__      :      Optional[Any]         =		16
						lowerCAmelCase__      :      Any         =		24
						lowerCAmelCase__      :      Dict         =		768
						lowerCAmelCase__      :      List[str]         =		3_072

			if model_name == "xclip-large-patch14-16-frames":
						lowerCAmelCase__      :      str         =		336

			lowerCAmelCase__      :      Any         =		XCLIPConfig.from_text_vision_configs(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )

			if "large" in model_name:
						lowerCAmelCase__      :      List[Any]         =		768

			return config
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		List[Any]:
			# text encoder
			if name == "token_embedding.weight":
						lowerCAmelCase__      :      Optional[Any]         =		name.replace('token_embedding.weight'       ,							'text_model.embeddings.token_embedding.weight'     )
			if name == "positional_embedding":
						lowerCAmelCase__      :      int         =		name.replace('positional_embedding'       ,							'text_model.embeddings.position_embedding.weight'     )
			if "ln_1" in name:
						lowerCAmelCase__      :      Optional[Any]         =		name.replace('ln_1'       ,							'layer_norm1'     )
			if "ln_2" in name:
						lowerCAmelCase__      :      List[str]         =		name.replace('ln_2'       ,							'layer_norm2'     )
			if "c_fc" in name:
						lowerCAmelCase__      :      Tuple         =		name.replace('c_fc'       ,							'fc1'     )
			if "c_proj" in name:
						lowerCAmelCase__      :      Optional[Any]         =		name.replace('c_proj'       ,							'fc2'     )
			if name.startswith('transformer.resblocks'     ):
						lowerCAmelCase__      :      List[str]         =		name.replace('transformer.resblocks'       ,							'text_model.encoder.layers'     )
			if "attn.out_proj" in name and "message" not in name:
						lowerCAmelCase__      :      Optional[Any]         =		name.replace('attn.out_proj'       ,							'self_attn.out_proj'     )
			if "ln_final" in name:
						lowerCAmelCase__      :      List[str]         =		name.replace('ln_final'       ,							'text_model.final_layer_norm'     )
			# visual encoder
			if name == "visual.class_embedding":
						lowerCAmelCase__      :      Dict         =		name.replace('visual.class_embedding'       ,							'vision_model.embeddings.class_embedding'     )
			if name == "visual.positional_embedding":
						lowerCAmelCase__      :      Any         =		name.replace('visual.positional_embedding'       ,							'vision_model.embeddings.position_embedding.weight'     )
			if name.startswith('visual.transformer.resblocks'     ):
						lowerCAmelCase__      :      Union[str, Any]         =		name.replace('visual.transformer.resblocks'       ,							'vision_model.encoder.layers'     )
			if "visual.conv1" in name:
						lowerCAmelCase__      :      Dict         =		name.replace('visual.conv1'       ,							'vision_model.embeddings.patch_embedding'     )
			if "visual.ln_pre" in name:
						lowerCAmelCase__      :      Union[str, Any]         =		name.replace('visual.ln_pre'       ,							'vision_model.pre_layernorm'     )
			if "visual.ln_post" in name:
						lowerCAmelCase__      :      Tuple         =		name.replace('visual.ln_post'       ,							'vision_model.post_layernorm'     )
			if "visual.proj" in name:
						lowerCAmelCase__      :      Optional[Any]         =		name.replace('visual.proj'       ,							'visual_projection.weight'     )
			if "text_projection" in name:
						lowerCAmelCase__      :      Any         =		name.replace('text_projection'       ,							'text_projection.weight'     )
			# things on top
			if "prompts_visual_proj" in name:
						lowerCAmelCase__      :      str         =		name.replace('prompts_visual_proj'       ,							'prompts_visual_projection'     )
			if "prompts_visual_ln" in name:
						lowerCAmelCase__      :      Tuple         =		name.replace('prompts_visual_ln'       ,							'prompts_visual_layernorm'     )
			# mit
			if name == "mit.positional_embedding":
						lowerCAmelCase__      :      Dict         =		name.replace('positional'       ,							'position'     )
			if name.startswith('mit.resblocks'     ):
						lowerCAmelCase__      :      List[str]         =		name.replace('mit.resblocks'       ,							'mit.encoder.layers'     )
			# prompts generator
			if name.startswith('prompts_generator.norm'     ):
						lowerCAmelCase__      :      List[Any]         =		name.replace('prompts_generator.norm'       ,							'prompts_generator.layernorm'     )

			return name
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		Union[str, Any]:
			for key in orig_state_dict.copy().keys():
						lowerCAmelCase__      :      Any         =		orig_state_dict.pop(SCREAMING_SNAKE_CASE_     )

						if "attn.in_proj" in key:
									lowerCAmelCase__      :      str         =		key.split('.'     )
									if key.startswith('visual'     ):
												lowerCAmelCase__      :      str         =		key_split[3]
												lowerCAmelCase__      :      Optional[Any]         =		config.vision_config.hidden_size
												if "message_attn" in key:
															if "weight" in key:
																		lowerCAmelCase__      :      Any         =		val[
																		    :dim, :
																		]
																		lowerCAmelCase__      :      Optional[Any]         =		val[
																		    dim : dim * 2, :
																		]
																		lowerCAmelCase__      :      int         =		val[
																		    -dim:, :
																		]
															else:
																		lowerCAmelCase__      :      str         =		val[
																		    :dim
																		]
																		lowerCAmelCase__      :      Union[str, Any]         =		val[
																		    dim : dim * 2
																		]
																		lowerCAmelCase__      :      Any         =		val[
																		    -dim:
																		]
												else:
															if "weight" in key:
																		lowerCAmelCase__      :      str         =		val[
																		    :dim, :
																		]
																		lowerCAmelCase__      :      Tuple         =		val[
																		    dim : dim * 2, :
																		]
																		lowerCAmelCase__      :      int         =		val[
																		    -dim:, :
																		]
															else:
																		lowerCAmelCase__      :      List[Any]         =		val[:dim]
																		lowerCAmelCase__      :      Optional[Any]         =		val[
																		    dim : dim * 2
																		]
																		lowerCAmelCase__      :      int         =		val[-dim:]
									elif key.startswith('mit'     ):
												lowerCAmelCase__      :      Dict         =		key_split[2]
												lowerCAmelCase__      :      str         =		config.vision_config.mit_hidden_size
												if "weight" in key:
															lowerCAmelCase__      :      List[Any]         =		val[:dim, :]
															lowerCAmelCase__      :      List[Any]         =		val[dim : dim * 2, :]
															lowerCAmelCase__      :      Any         =		val[-dim:, :]
												else:
															lowerCAmelCase__      :      int         =		val[:dim]
															lowerCAmelCase__      :      Optional[Any]         =		val[dim : dim * 2]
															lowerCAmelCase__      :      List[str]         =		val[-dim:]
									else:
												lowerCAmelCase__      :      int         =		key_split[2]
												lowerCAmelCase__      :      Optional[Any]         =		config.text_config.hidden_size
												if "weight" in key:
															lowerCAmelCase__      :      int         =		val[:dim, :]
															lowerCAmelCase__      :      List[str]         =		val[
															    dim : dim * 2, :
															]
															lowerCAmelCase__      :      Dict         =		val[-dim:, :]
												else:
															lowerCAmelCase__      :      List[str]         =		val[:dim]
															lowerCAmelCase__      :      int         =		val[
															    dim : dim * 2
															]
															lowerCAmelCase__      :      Optional[int]         =		val[-dim:]
						else:
									lowerCAmelCase__      :      Optional[int]         =		rename_key(SCREAMING_SNAKE_CASE_     )
									if new_key_name in ["visual_projection.weight", "text_projection.weight"]:
												lowerCAmelCase__      :      List[str]         =		val.T
									lowerCAmelCase__      :      List[str]         =		val

			return orig_state_dict
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		Dict:
			if num_frames == 8:
						lowerCAmelCase__      :      Tuple         =		'eating_spaghetti_8_frames.npy'
			elif num_frames == 16:
						lowerCAmelCase__      :      Dict         =		'eating_spaghetti.npy'
			elif num_frames == 32:
						lowerCAmelCase__      :      List[Any]         =		'eating_spaghetti_32_frames.npy'
			lowerCAmelCase__      :      Dict         =		hf_hub_download(
			    repo_id='hf-internal-testing/spaghetti-video'       ,							filename=SCREAMING_SNAKE_CASE_       ,							repo_type='dataset'       ,							)
			lowerCAmelCase__      :      Optional[int]         =		np.load(SCREAMING_SNAKE_CASE_     )
			return list(SCREAMING_SNAKE_CASE_     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_=None       ,							SCREAMING_SNAKE_CASE_=False     )				->		str:
			lowerCAmelCase__      :      Optional[int]         =		{
			    # fully supervised kinetics-400 checkpoints
			    'xclip-base-patch32': 'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_32_8.pth',
			    'xclip-base-patch32-16-frames': (
			        'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_32_16.pth'
			    ),
			    'xclip-base-patch16': 'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_16_8.pth',
			    'xclip-base-patch16-16-frames': (
			        'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_16_16.pth'
			    ),
			    'xclip-large-patch14': 'https://drive.google.com/u/0/uc?id=1NUOImq0o5DlQTST17iIP3vG7DgmHQuCx&amp;export=download&amp;confirm=t&amp;uuid=b26caedc-88e2-473e-830a-9d158b653cdb',
			    'xclip-large-patch14-16-frames': 'https://drive.google.com/u/0/uc?id=1FOYgnJc097OJ4lGwtRCCydQyVPJEOH7d&amp;export=download&amp;confirm=t&amp;uuid=538fa810-e671-4050-b385-9a623f89804f',
			    # fully supervised kinetics-600 checkpoints
			    'xclip-base-patch16-kinetics-600': (
			        'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k600_16_8.pth'
			    ),
			    'xclip-base-patch16-kinetics-600-16-frames': (
			        'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k600_16_16.pth'
			    ),
			    'xclip-large-patch14-kinetics-600': 'https://drive.google.com/u/0/uc?id=1FV8C1INuM91sLAN4ImjzePLIlpMSihwV&amp;export=download&amp;confirm=t&amp;uuid=141d4977-4a65-44ae-864f-4b0c19f838be',
			    # few shot
			    'xclip-base-patch16-hmdb-2-shot': (
			        'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_2.pth'
			    ),
			    'xclip-base-patch16-hmdb-4-shot': (
			        'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_4.pth'
			    ),
			    'xclip-base-patch16-hmdb-8-shot': (
			        'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_8.pth'
			    ),
			    'xclip-base-patch16-hmdb-16-shot': (
			        'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_16.pth'
			    ),
			    'xclip-base-patch16-ucf-2-shot': (
			        'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_2.pth'
			    ),
			    'xclip-base-patch16-ucf-4-shot': (
			        'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_4.pth'
			    ),
			    'xclip-base-patch16-ucf-8-shot': (
			        'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_8.pth'
			    ),
			    'xclip-base-patch16-ucf-16-shot': (
			        'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_16.pth'
			    ),
			    # zero shot
			    'xclip-base-patch16-zero-shot': 'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/zero.pth',
			}

			lowerCAmelCase__      :      Dict         =		model_to_url[model_name]
			lowerCAmelCase__      :      List[str]         =		8
			if "16-frames" in model_name:
						lowerCAmelCase__      :      Any         =		16
			elif "shot" in model_name:
						lowerCAmelCase__      :      str         =		32

			lowerCAmelCase__      :      str         =		get_xclip_config(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )
			lowerCAmelCase__      :      Any         =		XCLIPModel(SCREAMING_SNAKE_CASE_     )
			model.eval()

			if "drive" in checkpoint_url:
						lowerCAmelCase__      :      Any         =		'pytorch_model.bin'
						gdown.cached_download(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							quiet=SCREAMING_SNAKE_CASE_     )
						lowerCAmelCase__      :      Tuple         =		torch.load(SCREAMING_SNAKE_CASE_       ,							map_location='cpu'     )['model']
			else:
						lowerCAmelCase__      :      Optional[Any]         =		torch.hub.load_state_dict_from_url(SCREAMING_SNAKE_CASE_     )['model']

			lowerCAmelCase__      :      Optional[Any]         =		convert_state_dict(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )

			lowerCAmelCase__      :      Optional[int]         =		XCLIPModel(SCREAMING_SNAKE_CASE_     )
			lowerCAmelCase__      ,				lowerCAmelCase__      :      str         =		model.load_state_dict(SCREAMING_SNAKE_CASE_       ,							strict=SCREAMING_SNAKE_CASE_     )
			assert missing_keys == ["text_model.embeddings.position_ids", "vision_model.embeddings.position_ids"]
			model.eval()

			lowerCAmelCase__      :      Optional[int]         =		336 if model_name == 'xclip-large-patch14-16-frames' else 224
			lowerCAmelCase__      :      List[str]         =		VideoMAEImageProcessor(size=SCREAMING_SNAKE_CASE_     )
			lowerCAmelCase__      :      Tuple         =		CLIPTokenizer.from_pretrained('openai/clip-vit-base-patch32'     )
			lowerCAmelCase__      :      List[Any]         =		CLIPTokenizerFast.from_pretrained('openai/clip-vit-base-patch32'     )
			lowerCAmelCase__      :      Optional[Any]         =		XCLIPProcessor(image_processor=SCREAMING_SNAKE_CASE_       ,							tokenizer=SCREAMING_SNAKE_CASE_     )

			lowerCAmelCase__      :      List[Any]         =		prepare_video(SCREAMING_SNAKE_CASE_     )
			lowerCAmelCase__      :      str         =		processor(
			    text=['playing sports', 'eating spaghetti', 'go shopping']       ,							videos=SCREAMING_SNAKE_CASE_       ,							return_tensors='pt'       ,							padding=SCREAMING_SNAKE_CASE_     )

			print('Shape of pixel values:'       ,							inputs.pixel_values.shape     )

			with torch.no_grad():
						lowerCAmelCase__      :      Union[str, Any]         =		model(**SCREAMING_SNAKE_CASE_     )

			# Verify outputs
			lowerCAmelCase__      :      Optional[Any]         =		outputs.logits_per_video
			lowerCAmelCase__      :      Tuple         =		logits_per_video.softmax(dim=1     )
			print('Probs:'       ,							SCREAMING_SNAKE_CASE_     )
			# kinetics-400
			if model_name == "xclip-base-patch32":
						lowerCAmelCase__      :      str         =		torch.tensor([[0.0019, 0.9951, 0.0030]]     )
			elif model_name == "xclip-base-patch32-16-frames":
						lowerCAmelCase__      :      Dict         =		torch.tensor([[7.0999e-04, 9.9883e-01, 4.5580e-04]]     )
			elif model_name == "xclip-base-patch16":
						lowerCAmelCase__      :      int         =		torch.tensor([[0.0083, 0.9681, 0.0236]]     )
			elif model_name == "xclip-base-patch16-16-frames":
						lowerCAmelCase__      :      Any         =		torch.tensor([[7.6937e-04, 9.9728e-01, 1.9473e-03]]     )
			elif model_name == "xclip-large-patch14":
						lowerCAmelCase__      :      Any         =		torch.tensor([[0.0062, 0.9864, 0.0075]]     )
			elif model_name == "xclip-large-patch14-16-frames":
						lowerCAmelCase__      :      Union[str, Any]         =		torch.tensor([[3.3877e-04, 9.9937e-01, 2.8888e-04]]     )
			# kinetics-600
			elif model_name == "xclip-base-patch16-kinetics-600":
						lowerCAmelCase__      :      Optional[int]         =		torch.tensor([[0.0555, 0.8914, 0.0531]]     )
			elif model_name == "xclip-base-patch16-kinetics-600-16-frames":
						lowerCAmelCase__      :      int         =		torch.tensor([[3.8554e-04, 9.9929e-01, 3.2754e-04]]     )
			elif model_name == "xclip-large-patch14-kinetics-600":
						lowerCAmelCase__      :      Dict         =		torch.tensor([[0.0036, 0.9920, 0.0045]]     )
			# few shot
			elif model_name == "xclip-base-patch16-hmdb-2-shot":
						lowerCAmelCase__      :      Optional[Any]         =		torch.tensor([[7.1890e-06, 9.9994e-01, 5.6559e-05]]     )
			elif model_name == "xclip-base-patch16-hmdb-4-shot":
						lowerCAmelCase__      :      Tuple         =		torch.tensor([[1.0320e-05, 9.9993e-01, 6.2435e-05]]     )
			elif model_name == "xclip-base-patch16-hmdb-8-shot":
						lowerCAmelCase__      :      int         =		torch.tensor([[4.1377e-06, 9.9990e-01, 9.8386e-05]]     )
			elif model_name == "xclip-base-patch16-hmdb-16-shot":
						lowerCAmelCase__      :      Optional[Any]         =		torch.tensor([[4.1347e-05, 9.9962e-01, 3.3411e-04]]     )
			elif model_name == "xclip-base-patch16-ucf-2-shot":
						lowerCAmelCase__      :      str         =		torch.tensor([[8.5857e-05, 9.9928e-01, 6.3291e-04]]     )
			elif model_name == "xclip-base-patch16-ucf-4-shot":
						lowerCAmelCase__      :      Optional[Any]         =		torch.tensor([[8.5857e-05, 9.9928e-01, 6.3291e-04]]     )
			elif model_name == "xclip-base-patch16-ucf-8-shot":
						lowerCAmelCase__      :      int         =		torch.tensor([[0.0027, 0.9904, 0.0070]]     )
			elif model_name == "xclip-base-patch16-ucf-16-shot":
						lowerCAmelCase__      :      Union[str, Any]         =		torch.tensor([[9.8219e-04, 9.9593e-01, 3.0863e-03]]     )
			# zero shot
			elif model_name == "xclip-base-patch16-zero-shot":
						lowerCAmelCase__      :      List[Any]         =		torch.tensor([[3.5082e-04, 9.9785e-01, 1.7966e-03]]     )
			else:
						raise ValueError(F'''Model name {model_name} not supported'''     )
			assert torch.allclose(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							atol=1e-3     )
			print('Looks ok!'     )

			if pytorch_dump_folder_path is not None:
						print(F'''Saving model {model_name} to {pytorch_dump_folder_path}'''     )
						model.save_pretrained(SCREAMING_SNAKE_CASE_     )

			if push_to_hub:
						print('Pushing model, processor and slow tokenizer files to the hub...'     )
						model.push_to_hub(SCREAMING_SNAKE_CASE_       ,							organization='nielsr'     )
						processor.push_to_hub(SCREAMING_SNAKE_CASE_       ,							organization='nielsr'     )
						slow_tokenizer.push_to_hub(SCREAMING_SNAKE_CASE_       ,							organization='nielsr'     )


if __name__ == "__main__":
					lowerCamelCase__          =		argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    """--model_name""",
					    default="""xclip-base-patch32""",
					    type=str,
					    help="""Name of the model.""",
					)
					parser.add_argument(
					    """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
					)
					parser.add_argument(
					    """--push_to_hub""", action="""store_true""", help="""Whether or not to push the converted model to the 🤗 hub."""
					)

					lowerCamelCase__          =		parser.parse_args()
					convert_xclip_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)

import json
import os
import shutil
import tempfile
from unittest import TestCase

from transformers import BartTokenizer, BartTokenizerFast, DPRQuestionEncoderTokenizer, DPRQuestionEncoderTokenizerFast
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_tokenizers, require_torch, slow
from transformers.utils import is_datasets_available, is_faiss_available, is_torch_available


if is_torch_available() and is_datasets_available() and is_faiss_available():
					from transformers.models.rag.configuration_rag import RagConfig
					from transformers.models.rag.tokenization_rag import RagTokenizer







@require_faiss
@require_torch
class      A__							(     __magic_name__	):






				def 							_lowerCamelCase		(						self					:      Tuple   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[Any]         =		tempfile.mkdtemp()
							lowerCAmelCase__      :      Dict         =		8

							# DPR tok
							lowerCAmelCase__      :      str         =		[
							    '[UNK]',
							    '[CLS]',
							    '[SEP]',
							    '[PAD]',
							    '[MASK]',
							    'want',
							    '##want',
							    '##ed',
							    'wa',
							    'un',
							    'runn',
							    '##ing',
							    ',',
							    'low',
							    'lowest',
							]
							lowerCAmelCase__      :      List[str]         =		os.path.join(self.tmpdirname    ,     'dpr_tokenizer'   )
							os.makedirs(a    ,     exist_ok=a   )
							lowerCAmelCase__      :      Union[str, Any]         =		os.path.join(a    ,     DPR_VOCAB_FILES_NAMES['vocab_file']   )
							with open(self.vocab_file    ,     'w'    ,     encoding='utf-8'   ) as vocab_writer:
										vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]   )   )

							# BART tok
							lowerCAmelCase__      :      Optional[Any]         =		[
							    'l',
							    'o',
							    'w',
							    'e',
							    'r',
							    's',
							    't',
							    'i',
							    'd',
							    'n',
							    '\u0120',
							    '\u0120l',
							    '\u0120n',
							    '\u0120lo',
							    '\u0120low',
							    'er',
							    '\u0120lowest',
							    '\u0120newer',
							    '\u0120wider',
							    '<unk>',
							]
							lowerCAmelCase__      :      str         =		dict(zip(a    ,     range(len(a   )   )   )   )
							lowerCAmelCase__      :      int         =		['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
							lowerCAmelCase__      :      Dict         =		{'unk_token': '<unk>'}

							lowerCAmelCase__      :      Dict         =		os.path.join(self.tmpdirname    ,     'bart_tokenizer'   )
							os.makedirs(a    ,     exist_ok=a   )
							lowerCAmelCase__      :      Optional[Any]         =		os.path.join(a    ,     BART_VOCAB_FILES_NAMES['vocab_file']   )
							lowerCAmelCase__      :      Any         =		os.path.join(a    ,     BART_VOCAB_FILES_NAMES['merges_file']   )
							with open(self.vocab_file    ,     'w'    ,     encoding='utf-8'   ) as fp:
										fp.write(json.dumps(a   ) + '\n'   )
							with open(self.merges_file    ,     'w'    ,     encoding='utf-8'   ) as fp:
										fp.write('\n'.join(a   )   )






				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname    ,     'dpr_tokenizer'   )   )






				def 							_lowerCamelCase		(						self					:      Any   ):






							'''simple docstring'''







							return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname    ,     'bart_tokenizer'   )   )






				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							shutil.rmtree(self.tmpdirname   )






				@require_tokenizers
				def 							_lowerCamelCase		(						self					:      Any   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Union[str, Any]         =		os.path.join(self.tmpdirname    ,     'rag_tokenizer'   )
							lowerCAmelCase__      :      Any         =		RagConfig(question_encoder=DPRConfig().to_dict()    ,     generator=BartConfig().to_dict()   )
							lowerCAmelCase__      :      Any         =		RagTokenizer(question_encoder=self.get_dpr_tokenizer()    ,     generator=self.get_bart_tokenizer()   )
							rag_config.save_pretrained(a   )
							rag_tokenizer.save_pretrained(a   )
							lowerCAmelCase__      :      List[str]         =		RagTokenizer.from_pretrained(a    ,     config=a   )
							self.assertIsInstance(new_rag_tokenizer.question_encoder    ,     a   )
							self.assertEqual(new_rag_tokenizer.question_encoder.get_vocab()    ,     rag_tokenizer.question_encoder.get_vocab()   )
							self.assertIsInstance(new_rag_tokenizer.generator    ,     a   )
							self.assertEqual(new_rag_tokenizer.generator.get_vocab()    ,     rag_tokenizer.generator.get_vocab()   )






				@slow
				def 							_lowerCamelCase		(						self					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Union[str, Any]         =		RagTokenizer.from_pretrained('facebook/rag-token-nq'   )
							lowerCAmelCase__      :      Any         =		[
							    'who got the first nobel prize in physics',
							    'when is the next deadpool movie being released',
							    'which mode is used for short wave broadcast service',
							    'who is the owner of reading football club',
							    'when is the next scandal episode coming out',
							    'when is the last time the philadelphia won the superbowl',
							    'what is the most current adobe flash player version',
							    'how many episodes are there in dragon ball z',
							    'what is the first step in the evolution of the eye',
							    'where is gall bladder situated in human body',
							    'what is the main mineral in lithium batteries',
							    'who is the president of usa right now',
							    'where do the greasers live in the outsiders',
							    'panda is a national animal of which country',
							    'what is the name of manchester united stadium',
							]
							lowerCAmelCase__      :      List[str]         =		tokenizer(a   )
							self.assertIsNotNone(a   )






				@slow
				def 							_lowerCamelCase		(						self					:      List[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[str]         =		RagTokenizer.from_pretrained('facebook/rag-sequence-nq'   )
							lowerCAmelCase__      :      List[str]         =		[
							    'who got the first nobel prize in physics',
							    'when is the next deadpool movie being released',
							    'which mode is used for short wave broadcast service',
							    'who is the owner of reading football club',
							    'when is the next scandal episode coming out',
							    'when is the last time the philadelphia won the superbowl',
							    'what is the most current adobe flash player version',
							    'how many episodes are there in dragon ball z',
							    'what is the first step in the evolution of the eye',
							    'where is gall bladder situated in human body',
							    'what is the main mineral in lithium batteries',
							    'who is the president of usa right now',
							    'where do the greasers live in the outsiders',
							    'panda is a national animal of which country',
							    'what is the name of manchester united stadium',
							]
							lowerCAmelCase__      :      int         =		tokenizer(a   )
							self.assertIsNotNone(a   )