






import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import cached_download, hf_hub_download, hf_hub_url
from PIL import Image

from transformers import DetaConfig, DetaForObjectDetection, DetaImageProcessor, SwinConfig
from transformers.utils import logging


logging.set_verbosity_info()
UpperCamelCase 				=	logging.get_logger(__name__)

def        __lowerCamelCase			(		snake_case__		)							->  List[Any]:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE	      =  SwinConfig(
						    embed_dim=1_92       ,depths=(2, 2, 18, 2)       ,num_heads=(6, 12, 24, 48)       ,window_size=12       ,out_features=["""stage2""", """stage3""", """stage4"""]       ,)

						_SCREAMING_SNAKE_CASE	      =  DetaConfig(
						    backbone_config=a_       ,num_queries=9_00       ,encoder_ffn_dim=20_48       ,decoder_ffn_dim=20_48       ,num_feature_levels=5       ,assign_first_stage=a_       ,with_box_refine=a_       ,two_stage=a_       ,)

						# set labels
						_SCREAMING_SNAKE_CASE	      =  """huggingface/label-files"""
						if "o365" in model_name:
												_SCREAMING_SNAKE_CASE	      =  3_66
												_SCREAMING_SNAKE_CASE	      =  """object365-id2label.json"""
						else:
												_SCREAMING_SNAKE_CASE	      =  91
												_SCREAMING_SNAKE_CASE	      =  """coco-detection-id2label.json"""

						_SCREAMING_SNAKE_CASE	      =  num_labels
						_SCREAMING_SNAKE_CASE	      =  json.load(open(cached_download(hf_hub_url(a_       ,a_       ,repo_type="""dataset"""		)		)       ,"""r"""		)		)
						_SCREAMING_SNAKE_CASE	      =  {int(a_		): v for k, v in idalabel.items()}
						_SCREAMING_SNAKE_CASE	      =  idalabel
						_SCREAMING_SNAKE_CASE	      =  {v: k for k, v in idalabel.items()}

						return config

def        __lowerCamelCase			(		snake_case__		)							->  List[Any]:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE	      =  []

						# stem
						# fmt: off
						rename_keys.append(("""backbone.0.body.patch_embed.proj.weight""", """model.backbone.model.embeddings.patch_embeddings.projection.weight""")		)
						rename_keys.append(("""backbone.0.body.patch_embed.proj.bias""", """model.backbone.model.embeddings.patch_embeddings.projection.bias""")		)
						rename_keys.append(("""backbone.0.body.patch_embed.norm.weight""", """model.backbone.model.embeddings.norm.weight""")		)
						rename_keys.append(("""backbone.0.body.patch_embed.norm.bias""", """model.backbone.model.embeddings.norm.bias""")		)
						# stages
						for i in range(len(config.backbone_config.depths		)		):
												for j in range(config.backbone_config.depths[i]		):
																		rename_keys.append((F'backbone.0.body.layers.{i}.blocks.{j}.norm1.weight', F'model.backbone.model.encoder.layers.{i}.blocks.{j}.layernorm_before.weight')		)
																		rename_keys.append((F'backbone.0.body.layers.{i}.blocks.{j}.norm1.bias', F'model.backbone.model.encoder.layers.{i}.blocks.{j}.layernorm_before.bias')		)
																		rename_keys.append((F'backbone.0.body.layers.{i}.blocks.{j}.attn.relative_position_bias_table', F'model.backbone.model.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_bias_table')		)
																		rename_keys.append((F'backbone.0.body.layers.{i}.blocks.{j}.attn.relative_position_index', F'model.backbone.model.encoder.layers.{i}.blocks.{j}.attention.self.relative_position_index')		)
																		rename_keys.append((F'backbone.0.body.layers.{i}.blocks.{j}.attn.proj.weight', F'model.backbone.model.encoder.layers.{i}.blocks.{j}.attention.output.dense.weight')		)
																		rename_keys.append((F'backbone.0.body.layers.{i}.blocks.{j}.attn.proj.bias', F'model.backbone.model.encoder.layers.{i}.blocks.{j}.attention.output.dense.bias')		)
																		rename_keys.append((F'backbone.0.body.layers.{i}.blocks.{j}.norm2.weight', F'model.backbone.model.encoder.layers.{i}.blocks.{j}.layernorm_after.weight')		)
																		rename_keys.append((F'backbone.0.body.layers.{i}.blocks.{j}.norm2.bias', F'model.backbone.model.encoder.layers.{i}.blocks.{j}.layernorm_after.bias')		)
																		rename_keys.append((F'backbone.0.body.layers.{i}.blocks.{j}.mlp.fc1.weight', F'model.backbone.model.encoder.layers.{i}.blocks.{j}.intermediate.dense.weight')		)
																		rename_keys.append((F'backbone.0.body.layers.{i}.blocks.{j}.mlp.fc1.bias', F'model.backbone.model.encoder.layers.{i}.blocks.{j}.intermediate.dense.bias')		)
																		rename_keys.append((F'backbone.0.body.layers.{i}.blocks.{j}.mlp.fc2.weight', F'model.backbone.model.encoder.layers.{i}.blocks.{j}.output.dense.weight')		)
																		rename_keys.append((F'backbone.0.body.layers.{i}.blocks.{j}.mlp.fc2.bias', F'model.backbone.model.encoder.layers.{i}.blocks.{j}.output.dense.bias')		)

												if i < 3:
																		rename_keys.append((F'backbone.0.body.layers.{i}.downsample.reduction.weight', F'model.backbone.model.encoder.layers.{i}.downsample.reduction.weight')		)
																		rename_keys.append((F'backbone.0.body.layers.{i}.downsample.norm.weight', F'model.backbone.model.encoder.layers.{i}.downsample.norm.weight')		)
																		rename_keys.append((F'backbone.0.body.layers.{i}.downsample.norm.bias', F'model.backbone.model.encoder.layers.{i}.downsample.norm.bias')		)

						rename_keys.append(("""backbone.0.body.norm1.weight""", """model.backbone.model.hidden_states_norms.stage2.weight""")		)
						rename_keys.append(("""backbone.0.body.norm1.bias""", """model.backbone.model.hidden_states_norms.stage2.bias""")		)
						rename_keys.append(("""backbone.0.body.norm2.weight""", """model.backbone.model.hidden_states_norms.stage3.weight""")		)
						rename_keys.append(("""backbone.0.body.norm2.bias""", """model.backbone.model.hidden_states_norms.stage3.bias""")		)
						rename_keys.append(("""backbone.0.body.norm3.weight""", """model.backbone.model.hidden_states_norms.stage4.weight""")		)
						rename_keys.append(("""backbone.0.body.norm3.bias""", """model.backbone.model.hidden_states_norms.stage4.bias""")		)

						# transformer encoder
						for i in range(config.encoder_layers		):
												rename_keys.append((F'transformer.encoder.layers.{i}.self_attn.sampling_offsets.weight', F'model.encoder.layers.{i}.self_attn.sampling_offsets.weight')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.self_attn.sampling_offsets.bias', F'model.encoder.layers.{i}.self_attn.sampling_offsets.bias')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.self_attn.attention_weights.weight', F'model.encoder.layers.{i}.self_attn.attention_weights.weight')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.self_attn.attention_weights.bias', F'model.encoder.layers.{i}.self_attn.attention_weights.bias')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.self_attn.value_proj.weight', F'model.encoder.layers.{i}.self_attn.value_proj.weight')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.self_attn.value_proj.bias', F'model.encoder.layers.{i}.self_attn.value_proj.bias')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.self_attn.output_proj.weight', F'model.encoder.layers.{i}.self_attn.output_proj.weight')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.self_attn.output_proj.bias', F'model.encoder.layers.{i}.self_attn.output_proj.bias')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.norm1.weight', F'model.encoder.layers.{i}.self_attn_layer_norm.weight')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.norm1.bias', F'model.encoder.layers.{i}.self_attn_layer_norm.bias')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.linear1.weight', F'model.encoder.layers.{i}.fc1.weight')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.linear1.bias', F'model.encoder.layers.{i}.fc1.bias')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.linear2.weight', F'model.encoder.layers.{i}.fc2.weight')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.linear2.bias', F'model.encoder.layers.{i}.fc2.bias')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.norm2.weight', F'model.encoder.layers.{i}.final_layer_norm.weight')		)
												rename_keys.append((F'transformer.encoder.layers.{i}.norm2.bias', F'model.encoder.layers.{i}.final_layer_norm.bias')		)

						# transformer decoder
						for i in range(config.decoder_layers		):
												rename_keys.append((F'transformer.decoder.layers.{i}.cross_attn.sampling_offsets.weight', F'model.decoder.layers.{i}.encoder_attn.sampling_offsets.weight')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.cross_attn.sampling_offsets.bias', F'model.decoder.layers.{i}.encoder_attn.sampling_offsets.bias')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.cross_attn.attention_weights.weight', F'model.decoder.layers.{i}.encoder_attn.attention_weights.weight')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.cross_attn.attention_weights.bias', F'model.decoder.layers.{i}.encoder_attn.attention_weights.bias')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.cross_attn.value_proj.weight', F'model.decoder.layers.{i}.encoder_attn.value_proj.weight')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.cross_attn.value_proj.bias', F'model.decoder.layers.{i}.encoder_attn.value_proj.bias')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.cross_attn.output_proj.weight', F'model.decoder.layers.{i}.encoder_attn.output_proj.weight')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.cross_attn.output_proj.bias', F'model.decoder.layers.{i}.encoder_attn.output_proj.bias')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.norm1.weight', F'model.decoder.layers.{i}.encoder_attn_layer_norm.weight')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.norm1.bias', F'model.decoder.layers.{i}.encoder_attn_layer_norm.bias')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.self_attn.out_proj.weight', F'model.decoder.layers.{i}.self_attn.out_proj.weight')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.self_attn.out_proj.bias', F'model.decoder.layers.{i}.self_attn.out_proj.bias')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.norm2.weight', F'model.decoder.layers.{i}.self_attn_layer_norm.weight')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.norm2.bias', F'model.decoder.layers.{i}.self_attn_layer_norm.bias')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.linear1.weight', F'model.decoder.layers.{i}.fc1.weight')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.linear1.bias', F'model.decoder.layers.{i}.fc1.bias')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.linear2.weight', F'model.decoder.layers.{i}.fc2.weight')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.linear2.bias', F'model.decoder.layers.{i}.fc2.bias')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.norm3.weight', F'model.decoder.layers.{i}.final_layer_norm.weight')		)
												rename_keys.append((F'transformer.decoder.layers.{i}.norm3.bias', F'model.decoder.layers.{i}.final_layer_norm.bias')		)

						# fmt: on

						return rename_keys

def        __lowerCamelCase			(		snake_case__       ,snake_case__       ,snake_case__		)							->  Optional[Any]:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE	      =  dct.pop(a_		)
						_SCREAMING_SNAKE_CASE	      =  val

def        __lowerCamelCase			(		snake_case__       ,snake_case__		)							->  int:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE	      =  [int(backbone_config.embed_dim * 2**i		) for i in range(len(backbone_config.depths		)		)]
						for i in range(len(backbone_config.depths		)		):
												_SCREAMING_SNAKE_CASE	      =  num_features[i]
												for j in range(backbone_config.depths[i]		):
																		# fmt: off
																		# read in weights + bias of input projection layer (in original implementation, this is a single matrix + bias)
																		_SCREAMING_SNAKE_CASE	      =  state_dict.pop(F'backbone.0.body.layers.{i}.blocks.{j}.attn.qkv.weight'		)
																		_SCREAMING_SNAKE_CASE	      =  state_dict.pop(F'backbone.0.body.layers.{i}.blocks.{j}.attn.qkv.bias'		)
																		# next, add query, keys and values (in that order) to the state dict
																		_SCREAMING_SNAKE_CASE	      =  in_proj_weight[:dim, :]
																		_SCREAMING_SNAKE_CASE	      =  in_proj_bias[: dim]
																		_SCREAMING_SNAKE_CASE	      =  in_proj_weight[
																		    dim : dim * 2, :
																		]
																		_SCREAMING_SNAKE_CASE	      =  in_proj_bias[
																		    dim : dim * 2
																		]
																		_SCREAMING_SNAKE_CASE	      =  in_proj_weight[
																		    -dim :, :
																		]
																		_SCREAMING_SNAKE_CASE	      =  in_proj_bias[-dim :]
																		# fmt: on

def        __lowerCamelCase			(		snake_case__       ,snake_case__		)							->  Optional[int]:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE	      =  config.d_model
						for i in range(config.decoder_layers		):
												# read in weights + bias of input projection layer of self-attention
												_SCREAMING_SNAKE_CASE	      =  state_dict.pop(F'transformer.decoder.layers.{i}.self_attn.in_proj_weight'		)
												_SCREAMING_SNAKE_CASE	      =  state_dict.pop(F'transformer.decoder.layers.{i}.self_attn.in_proj_bias'		)
												# next, add query, keys and values (in that order) to the state dict
												_SCREAMING_SNAKE_CASE	      =  in_proj_weight[:hidden_size, :]
												_SCREAMING_SNAKE_CASE	      =  in_proj_bias[:hidden_size]
												_SCREAMING_SNAKE_CASE	      =  in_proj_weight[
												    hidden_size : hidden_size * 2, :
												]
												_SCREAMING_SNAKE_CASE	      =  in_proj_bias[hidden_size : hidden_size * 2]
												_SCREAMING_SNAKE_CASE	      =  in_proj_weight[-hidden_size:, :]
												_SCREAMING_SNAKE_CASE	      =  in_proj_bias[-hidden_size:]

def        __lowerCamelCase			(		)							->  str:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE	      =  """http://images.cocodataset.org/val2017/000000039769.jpg"""
						_SCREAMING_SNAKE_CASE	      =  Image.open(requests.get(a_       ,stream=a_		).raw		)

						return im

@torch.no_grad()
def        __lowerCamelCase			(		snake_case__       ,snake_case__       ,snake_case__		)							->  str:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE	      =  get_deta_config(a_		)

						# load original state dict
						if model_name == "deta-swin-large":
												_SCREAMING_SNAKE_CASE	      =  hf_hub_download(repo_id="""nielsr/deta-checkpoints"""       ,filename="""adet_swin_ft.pth"""		)
						elif model_name == "deta-swin-large-o365":
												_SCREAMING_SNAKE_CASE	      =  hf_hub_download(repo_id="""jozhang97/deta-swin-l-o365"""       ,filename="""deta_swin_pt_o365.pth"""		)
						else:
												raise ValueError(F'Model name {model_name} not supported'		)

						_SCREAMING_SNAKE_CASE	      =  torch.load(a_       ,map_location="""cpu"""		)["""model"""]

						# original state dict
						for name, param in state_dict.items():
												print(a_       ,param.shape		)

						# rename keys
						_SCREAMING_SNAKE_CASE	      =  create_rename_keys(a_		)
						for src, dest in rename_keys:
												rename_key(a_       ,a_       ,a_		)
						read_in_swin_q_k_v(a_       ,config.backbone_config		)
						read_in_decoder_q_k_v(a_       ,a_		)

						# fix some prefixes
						for key in state_dict.copy().keys():
												if "transformer.decoder.class_embed" in key or "transformer.decoder.bbox_embed" in key:
																		_SCREAMING_SNAKE_CASE	      =  state_dict.pop(a_		)
																		_SCREAMING_SNAKE_CASE	      =  val
												if "input_proj" in key:
																		_SCREAMING_SNAKE_CASE	      =  state_dict.pop(a_		)
																		_SCREAMING_SNAKE_CASE	      =  val
												if "level_embed" in key or "pos_trans" in key or "pix_trans" in key or "enc_output" in key:
																		_SCREAMING_SNAKE_CASE	      =  state_dict.pop(a_		)
																		_SCREAMING_SNAKE_CASE	      =  val

    # finally, create HuggingFace model and load state dict
						_SCREAMING_SNAKE_CASE	      =  DetaForObjectDetection(a_		)
						model.load_state_dict(a_		)
						model.eval()

						_SCREAMING_SNAKE_CASE	      =  """cuda""" if torch.cuda.is_available() else """cpu"""
						model.to(a_		)

						# load image processor
						_SCREAMING_SNAKE_CASE	      =  DetaImageProcessor(format="""coco_detection"""		)

						# verify our conversion on image
						_SCREAMING_SNAKE_CASE	      =  prepare_img()
						_SCREAMING_SNAKE_CASE	      =  processor(images=a_       ,return_tensors="""pt"""		)
						_SCREAMING_SNAKE_CASE	      =  encoding["""pixel_values"""]
						_SCREAMING_SNAKE_CASE	      =  model(pixel_values.to(a_		)		)

						# verify logits
						print("""Logits:"""       ,outputs.logits[0, :3, :3]		)
						print("""Boxes:"""       ,outputs.pred_boxes[0, :3, :3]		)
						if model_name == "deta-swin-large":
												_SCREAMING_SNAKE_CASE	      =  torch.tensor(
												    [[-7.6_308, -2.8_485, -5.3_737], [-7.2_037, -4.5_505, -4.8_027], [-7.2_943, -4.2_611, -4.6_617]]		)
												_SCREAMING_SNAKE_CASE	      =  torch.tensor([[0.4_987, 0.4_969, 0.9_999], [0.2_549, 0.5_498, 0.4_805], [0.5_498, 0.2_757, 0.0_569]]		)
						elif model_name == "deta-swin-large-o365":
												_SCREAMING_SNAKE_CASE	      =  torch.tensor(
												    [[-8.0_122, -3.5_720, -4.9_717], [-8.1_547, -3.6_886, -4.6_389], [-7.6_610, -3.6_194, -5.0_134]]		)
												_SCREAMING_SNAKE_CASE	      =  torch.tensor([[0.2_523, 0.5_549, 0.4_881], [0.7_715, 0.4_149, 0.4_601], [0.5_503, 0.2_753, 0.0_575]]		)
						assert torch.allclose(outputs.logits[0, :3, :3]       ,expected_logits.to(a_		)       ,atol=1e-4		)
						assert torch.allclose(outputs.pred_boxes[0, :3, :3]       ,expected_boxes.to(a_		)       ,atol=1e-4		)
						print("""Everything ok!"""		)

						if pytorch_dump_folder_path:
												# Save model and processor
												logger.info(F'Saving PyTorch model and processor to {pytorch_dump_folder_path}...'		)
												Path(a_		).mkdir(exist_ok=a_		)
												model.save_pretrained(a_		)
												processor.save_pretrained(a_		)

						# Push to hub
						if push_to_hub:
												print("""Pushing model and processor to hub..."""		)
												model.push_to_hub(F'jozhang97/{model_name}'		)
												processor.push_to_hub(F'jozhang97/{model_name}'		)


if __name__ == "__main__":
		UpperCamelCase 				=	argparse.ArgumentParser()

		parser.add_argument(
		    '''--model_name''',
		    type=str,
		    default='''deta-swin-large''',
		    choices=['''deta-swin-large''', '''deta-swin-large-o365'''],
		    help='''Name of the model you\'d like to convert.''',
		)
		parser.add_argument(
		    '''--pytorch_dump_folder_path''',
		    default=None,
		    type=str,
		    help='''Path to the folder to output PyTorch model.''',
		)
		parser.add_argument(
		    '''--push_to_hub''', action='''store_true''', help='''Whether or not to push the converted model to the 🤗 hub.'''
		)
		UpperCamelCase 				=	parser.parse_args()
		convert_deta_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)









import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel




class  UpperCamelCase_      :



	'''simple docstring'''




	def __init__(       self       ,			a       ,			a=13       ,			a=7       ,			a=True       ,			a=True       ,			a=True       ,			a=True       ,			a=99       ,			a=32       ,			a=5       ,			a=4       ,			a=4       ,			a="gelu"       ,			a=0.0       ,			a=0.1       ,			a=True       ,			a=5_12       ,			a=16       ,			a=2       ,			a=0.02       ,			a=3       ,			a=4       ,			a=None       ,			)       ->    Optional[Any]:
								snake_case_				=							parent
								snake_case_				=							batch_size
								snake_case_				=							seq_length
								snake_case_				=							is_training
								snake_case_				=							use_input_mask
								snake_case_				=							use_token_type_ids
								snake_case_				=							use_labels
								snake_case_				=							vocab_size
								snake_case_				=							hidden_size
								snake_case_				=							num_hidden_layers
								snake_case_				=							num_attention_heads
								snake_case_				=							intermediate_multiple_size
								snake_case_				=							hidden_act
								snake_case_				=							hidden_dropout
								snake_case_				=							attention_dropout
								snake_case_				=							weight_tying
								snake_case_				=							max_position_embeddings
								snake_case_				=							type_vocab_size
								snake_case_				=							type_sequence_label_size
								snake_case_				=							initializer_range
								snake_case_				=							num_labels
								snake_case_				=							num_choices
								snake_case_				=							scope




	def 							_UpperCamelCase			(       self					)       ->    List[Any]:
								snake_case_				=							ids_tensor([self.batch_size, self.seq_length]       ,			self.vocab_size					)

								snake_case_				=							None
								if self.use_input_mask:
															snake_case_				=							random_attention_mask([self.batch_size, self.seq_length]					)

								snake_case_				=							None
								if self.use_labels:
															snake_case_				=							ids_tensor([self.batch_size, self.seq_length]       ,			self.num_labels					)

								snake_case_				=							self.get_config()

								return config, input_ids, input_mask, token_labels




	def 							_UpperCamelCase			(       self					)       ->    Dict:
								return GPTNeoXJapaneseConfig(
								    vocab_size=self.vocab_size       ,			hidden_size=self.hidden_size       ,			num_hidden_layers=self.num_hidden_layers       ,			num_attention_heads=self.num_attention_heads       ,			intermediate_multiple_size=self.intermediate_multiple_size       ,			hidden_act=self.hidden_act       ,			hidden_dropout=self.hidden_dropout       ,			attention_dropout=self.attention_dropout       ,			weight_tying=self.weight_tying       ,			max_position_embeddings=self.max_position_embeddings       ,			type_vocab_size=self.type_vocab_size       ,			is_decoder=a       ,			initializer_range=self.initializer_range       ,			)




	def 							_UpperCamelCase			(       self					)       ->    int:
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.prepare_config_and_inputs()

								snake_case_				=							True

								return config, input_ids, input_mask, token_labels




	def 							_UpperCamelCase			(       self       ,			a       ,			a       ,			a					)       ->    Any:
								snake_case_				=							GPTNeoXJapaneseModel(config=a					)
								model.to(a					)
								model.eval()
								snake_case_				=							model(a       ,			attention_mask=a					)
								snake_case_				=							model(a					)
								self.parent.assertEqual(result.last_hidden_state.shape       ,			(self.batch_size, self.seq_length, self.hidden_size)					)




	def 							_UpperCamelCase			(       self       ,			a       ,			a       ,			a					)       ->    Union[str, Any]:
								snake_case_				=							True
								snake_case_				=							GPTNeoXJapaneseModel(a					)
								model.to(a					)
								model.eval()
								snake_case_				=							model(a       ,			attention_mask=a					)
								self.parent.assertEqual(result.last_hidden_state.shape       ,			(self.batch_size, self.seq_length, self.hidden_size)					)




	def 							_UpperCamelCase			(       self       ,			a       ,			a       ,			a       ,			a					)       ->    int:
								snake_case_				=							GPTNeoXJapaneseForCausalLM(config=a					)
								model.to(a					)
								model.eval()
								snake_case_				=							model(a       ,			attention_mask=a       ,			labels=a					)
								self.parent.assertEqual(result.logits.shape       ,			(self.batch_size, self.seq_length, self.vocab_size)					)




	def 							_UpperCamelCase			(       self       ,			a       ,			a       ,			a					)       ->    Tuple:
								snake_case_				=							True
								snake_case_				=							GPTNeoXJapaneseForCausalLM(config=a					)
								model.to(a					)
								model.eval()

								# first forward pass
								snake_case_				=							model(a       ,			attention_mask=a       ,			use_cache=a					)
								snake_case_				=							outputs.past_key_values

								# create hypothetical multiple next token and extent to next_input_ids
								snake_case_				=							ids_tensor((self.batch_size, 3)       ,			config.vocab_size					)
								snake_case_				=							ids_tensor((self.batch_size, 3)       ,			vocab_size=2					)

								# append to next input_ids and
								snake_case_				=							torch.cat([input_ids, next_tokens]       ,			dim=-1					)
								snake_case_				=							torch.cat([input_mask, next_mask]       ,			dim=-1					)

								snake_case_				=							model(a       ,			attention_mask=a       ,			output_hidden_states=a					)
								snake_case_				=							output_from_no_past['hidden_states'][0]
								snake_case_				=							model(
								    a       ,			attention_mask=a       ,			past_key_values=a       ,			output_hidden_states=a       ,			)['hidden_states'][0]

								# select random slice
								snake_case_				=							ids_tensor((1,)       ,			output_from_past.shape[-1]					).item()
								snake_case_				=							output_from_no_past[:, -3:, random_slice_idx].detach()
								snake_case_				=							output_from_past[:, :, random_slice_idx].detach()

								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]					)

								# test that outputs are equal for slice
								self.parent.assertTrue(torch.allclose(a       ,			a       ,			atol=1E-3					)					)






	def 							_UpperCamelCase			(       self					)       ->    Dict:
								snake_case_				=							self.prepare_config_and_inputs()
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							config_and_inputs
								snake_case_				=							{'input_ids': input_ids, 'attention_mask': input_mask}
								return config, inputs_dict






@require_torch
class  UpperCamelCase_      (			snake_case_       ,					snake_case_       ,					unittest.TestCase				):



	'''simple docstring'''




	lowerCAmelCase													= (GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
	lowerCAmelCase													= (GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
	lowerCAmelCase													= (
	    {'''feature-extraction''': GPTNeoXJapaneseModel, '''text-generation''': GPTNeoXJapaneseForCausalLM}
	    if is_torch_available()
	    else {}
	)
	lowerCAmelCase													= False
	lowerCAmelCase													= False
	lowerCAmelCase													= False
	lowerCAmelCase													= False




	def 							_UpperCamelCase			(       self					)       ->    List[Any]:
								snake_case_				=							GPTNeoXJapaneseModelTester(self					)
								snake_case_				=							ConfigTester(self       ,			config_class=a       ,			hidden_size=37					)




	def 							_UpperCamelCase			(       self					)       ->    str:
								self.config_tester.run_common_tests()




	def 							_UpperCamelCase			(       self					)       ->    Optional[Any]:
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(a       ,			a       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    Union[str, Any]:
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.model_tester.prepare_config_and_inputs_for_decoder()
								self.model_tester.create_and_check_model_as_decoder(a       ,			a       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    Optional[int]:
								# This regression test was failing with PyTorch < 1.3
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.model_tester.prepare_config_and_inputs_for_decoder()

								snake_case_				=							None

								self.model_tester.create_and_check_model_as_decoder(a       ,			a       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    Dict:
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_decoder_model_past_large_inputs(a       ,			a       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    List[Any]:
								snake_case_				=							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_causal_lm(*a					)






	@slow
	def 							_UpperCamelCase			(       self					)       ->    Any:
								snake_case_				=							'abeja/gpt-neox-japanese-2.7b'

								snake_case_				=							['データサイエンティストとは、', '100年後に必要とされる会社は、', 'フルリモートの環境で働くために必要なことは、', '国境の長いトンネルを抜けると', '美味しい日本食といえば、']

								snake_case_				=							[
								    'データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。',
								    '100年後に必要とされる会社は、「人」が中心の会社です。',
								    'フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。',
								    '国境の長いトンネルを抜けると、そこは雪国だった。',
								    '美味しい日本食といえば、やっぱりお寿司ですよね。',
								]

								snake_case_				=							GPTNeoXJapaneseTokenizer.from_pretrained(a					)
								snake_case_				=							GPTNeoXJapaneseForCausalLM.from_pretrained(a					)

								snake_case_				=							[]
								for prompt in prompts:
															snake_case_				=							tokenizer(a       ,			return_tensors='pt'					).input_ids
															snake_case_				=							model.generate(a       ,			max_length=50					)
															snake_case_				=							tokenizer.batch_decode(a       ,			skip_special_tokens=a					)
															predicted_outputs += generated_string
								self.assertListEqual(a       ,			a					)





