






import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel




class  UpperCamelCase_      :



	'''simple docstring'''




	def __init__(       self       ,			a       ,			a=13       ,			a=7       ,			a=True       ,			a=True       ,			a=True       ,			a=True       ,			a=99       ,			a=32       ,			a=5       ,			a=4       ,			a=4       ,			a="gelu"       ,			a=0.0       ,			a=0.1       ,			a=True       ,			a=5_12       ,			a=16       ,			a=2       ,			a=0.02       ,			a=3       ,			a=4       ,			a=None       ,			)       ->    Optional[Any]:
								snake_case_				=							parent
								snake_case_				=							batch_size
								snake_case_				=							seq_length
								snake_case_				=							is_training
								snake_case_				=							use_input_mask
								snake_case_				=							use_token_type_ids
								snake_case_				=							use_labels
								snake_case_				=							vocab_size
								snake_case_				=							hidden_size
								snake_case_				=							num_hidden_layers
								snake_case_				=							num_attention_heads
								snake_case_				=							intermediate_multiple_size
								snake_case_				=							hidden_act
								snake_case_				=							hidden_dropout
								snake_case_				=							attention_dropout
								snake_case_				=							weight_tying
								snake_case_				=							max_position_embeddings
								snake_case_				=							type_vocab_size
								snake_case_				=							type_sequence_label_size
								snake_case_				=							initializer_range
								snake_case_				=							num_labels
								snake_case_				=							num_choices
								snake_case_				=							scope




	def 							_UpperCamelCase			(       self					)       ->    List[Any]:
								snake_case_				=							ids_tensor([self.batch_size, self.seq_length]       ,			self.vocab_size					)

								snake_case_				=							None
								if self.use_input_mask:
															snake_case_				=							random_attention_mask([self.batch_size, self.seq_length]					)

								snake_case_				=							None
								if self.use_labels:
															snake_case_				=							ids_tensor([self.batch_size, self.seq_length]       ,			self.num_labels					)

								snake_case_				=							self.get_config()

								return config, input_ids, input_mask, token_labels




	def 							_UpperCamelCase			(       self					)       ->    Dict:
								return GPTNeoXJapaneseConfig(
								    vocab_size=self.vocab_size       ,			hidden_size=self.hidden_size       ,			num_hidden_layers=self.num_hidden_layers       ,			num_attention_heads=self.num_attention_heads       ,			intermediate_multiple_size=self.intermediate_multiple_size       ,			hidden_act=self.hidden_act       ,			hidden_dropout=self.hidden_dropout       ,			attention_dropout=self.attention_dropout       ,			weight_tying=self.weight_tying       ,			max_position_embeddings=self.max_position_embeddings       ,			type_vocab_size=self.type_vocab_size       ,			is_decoder=a       ,			initializer_range=self.initializer_range       ,			)




	def 							_UpperCamelCase			(       self					)       ->    int:
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.prepare_config_and_inputs()

								snake_case_				=							True

								return config, input_ids, input_mask, token_labels




	def 							_UpperCamelCase			(       self       ,			a       ,			a       ,			a					)       ->    Any:
								snake_case_				=							GPTNeoXJapaneseModel(config=a					)
								model.to(a					)
								model.eval()
								snake_case_				=							model(a       ,			attention_mask=a					)
								snake_case_				=							model(a					)
								self.parent.assertEqual(result.last_hidden_state.shape       ,			(self.batch_size, self.seq_length, self.hidden_size)					)




	def 							_UpperCamelCase			(       self       ,			a       ,			a       ,			a					)       ->    Union[str, Any]:
								snake_case_				=							True
								snake_case_				=							GPTNeoXJapaneseModel(a					)
								model.to(a					)
								model.eval()
								snake_case_				=							model(a       ,			attention_mask=a					)
								self.parent.assertEqual(result.last_hidden_state.shape       ,			(self.batch_size, self.seq_length, self.hidden_size)					)




	def 							_UpperCamelCase			(       self       ,			a       ,			a       ,			a       ,			a					)       ->    int:
								snake_case_				=							GPTNeoXJapaneseForCausalLM(config=a					)
								model.to(a					)
								model.eval()
								snake_case_				=							model(a       ,			attention_mask=a       ,			labels=a					)
								self.parent.assertEqual(result.logits.shape       ,			(self.batch_size, self.seq_length, self.vocab_size)					)




	def 							_UpperCamelCase			(       self       ,			a       ,			a       ,			a					)       ->    Tuple:
								snake_case_				=							True
								snake_case_				=							GPTNeoXJapaneseForCausalLM(config=a					)
								model.to(a					)
								model.eval()

								# first forward pass
								snake_case_				=							model(a       ,			attention_mask=a       ,			use_cache=a					)
								snake_case_				=							outputs.past_key_values

								# create hypothetical multiple next token and extent to next_input_ids
								snake_case_				=							ids_tensor((self.batch_size, 3)       ,			config.vocab_size					)
								snake_case_				=							ids_tensor((self.batch_size, 3)       ,			vocab_size=2					)

								# append to next input_ids and
								snake_case_				=							torch.cat([input_ids, next_tokens]       ,			dim=-1					)
								snake_case_				=							torch.cat([input_mask, next_mask]       ,			dim=-1					)

								snake_case_				=							model(a       ,			attention_mask=a       ,			output_hidden_states=a					)
								snake_case_				=							output_from_no_past['hidden_states'][0]
								snake_case_				=							model(
								    a       ,			attention_mask=a       ,			past_key_values=a       ,			output_hidden_states=a       ,			)['hidden_states'][0]

								# select random slice
								snake_case_				=							ids_tensor((1,)       ,			output_from_past.shape[-1]					).item()
								snake_case_				=							output_from_no_past[:, -3:, random_slice_idx].detach()
								snake_case_				=							output_from_past[:, :, random_slice_idx].detach()

								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]					)

								# test that outputs are equal for slice
								self.parent.assertTrue(torch.allclose(a       ,			a       ,			atol=1E-3					)					)






	def 							_UpperCamelCase			(       self					)       ->    Dict:
								snake_case_				=							self.prepare_config_and_inputs()
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							config_and_inputs
								snake_case_				=							{'input_ids': input_ids, 'attention_mask': input_mask}
								return config, inputs_dict






@require_torch
class  UpperCamelCase_      (			snake_case_       ,					snake_case_       ,					unittest.TestCase				):



	'''simple docstring'''




	lowerCAmelCase													= (GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
	lowerCAmelCase													= (GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
	lowerCAmelCase													= (
	    {'''feature-extraction''': GPTNeoXJapaneseModel, '''text-generation''': GPTNeoXJapaneseForCausalLM}
	    if is_torch_available()
	    else {}
	)
	lowerCAmelCase													= False
	lowerCAmelCase													= False
	lowerCAmelCase													= False
	lowerCAmelCase													= False




	def 							_UpperCamelCase			(       self					)       ->    List[Any]:
								snake_case_				=							GPTNeoXJapaneseModelTester(self					)
								snake_case_				=							ConfigTester(self       ,			config_class=a       ,			hidden_size=37					)




	def 							_UpperCamelCase			(       self					)       ->    str:
								self.config_tester.run_common_tests()




	def 							_UpperCamelCase			(       self					)       ->    Optional[Any]:
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(a       ,			a       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    Union[str, Any]:
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.model_tester.prepare_config_and_inputs_for_decoder()
								self.model_tester.create_and_check_model_as_decoder(a       ,			a       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    Optional[int]:
								# This regression test was failing with PyTorch < 1.3
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.model_tester.prepare_config_and_inputs_for_decoder()

								snake_case_				=							None

								self.model_tester.create_and_check_model_as_decoder(a       ,			a       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    Dict:
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_decoder_model_past_large_inputs(a       ,			a       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    List[Any]:
								snake_case_				=							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_causal_lm(*a					)






	@slow
	def 							_UpperCamelCase			(       self					)       ->    Any:
								snake_case_				=							'abeja/gpt-neox-japanese-2.7b'

								snake_case_				=							['データサイエンティストとは、', '100年後に必要とされる会社は、', 'フルリモートの環境で働くために必要なことは、', '国境の長いトンネルを抜けると', '美味しい日本食といえば、']

								snake_case_				=							[
								    'データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。',
								    '100年後に必要とされる会社は、「人」が中心の会社です。',
								    'フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。',
								    '国境の長いトンネルを抜けると、そこは雪国だった。',
								    '美味しい日本食といえば、やっぱりお寿司ですよね。',
								]

								snake_case_				=							GPTNeoXJapaneseTokenizer.from_pretrained(a					)
								snake_case_				=							GPTNeoXJapaneseForCausalLM.from_pretrained(a					)

								snake_case_				=							[]
								for prompt in prompts:
															snake_case_				=							tokenizer(a       ,			return_tensors='pt'					).input_ids
															snake_case_				=							model.generate(a       ,			max_length=50					)
															snake_case_				=							tokenizer.batch_decode(a       ,			skip_special_tokens=a					)
															predicted_outputs += generated_string
								self.assertListEqual(a       ,			a					)













import collections
import importlib.util
import os
import re
from pathlib import Path


lowercase  							=					"src/transformers"


# Matches is_xxx_available()
lowercase  							=					re.compile(r"is\_([a-z_]*)_available()")
# Catches a one-line _import_struct = {xxx}
lowercase  							=					re.compile(r"^_import_structure\s+=\s+\{([^\}]+)\}")
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
lowercase  							=					re.compile(r"\s+\"\S*\":\s+\[([^\]]*)\]")
# Catches a line if not is_foo_available
lowercase  							=					re.compile(r"^\s*if\s+not\s+is\_[a-z_]*\_available\(\)")
# Catches a line _import_struct["bla"].append("foo")
lowercase  							=					re.compile(r"^\s*_import_structure\[\"\S*\"\]\.append\(\"(\S*)\"\)")
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
lowercase  							=					re.compile(r"^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]")
# Catches a line with an object between quotes and a comma:     "MyModel",
lowercase  							=					re.compile("^\s+\"([^\"]+)\",")
# Catches a line with objects between brackets only:    ["foo", "bar"],
lowercase  							=					re.compile("^\s+\[([^\]]+)\]")
# Catches a line with from foo import bar, bla, boo
lowercase  							=					re.compile(r"\s+from\s+\S*\s+import\s+([^\(\s].*)\n")
# Catches a line with try:
lowercase  							=					re.compile(r"^\s*try:")
# Catches a line with else:
lowercase  							=					re.compile(r"^\s*else:")





def 						__UpperCAmelCase			(     a_):
							if _re_test_backend.search(a_) is None:
														return None
							snake_case_				=							[b[0] for b in _re_backend.findall(a_)]
							backends.sort()
							return "_and_".join(a_)





def 						__UpperCAmelCase			(     a_):
							with open(a_							,     'r'							,     encoding='utf-8'							,     newline='\n') as f:
														snake_case_				=							f.readlines()

							snake_case_				=							0
							while line_index < len(a_) and not lines[line_index].startswith('_import_structure = {'):
														line_index += 1

							# If this is a traditional init, just return.
							if line_index >= len(a_):
														return None

							# First grab the objects without a specific backend in _import_structure
							snake_case_				=							[]
							while not lines[line_index].startswith('if TYPE_CHECKING') and find_backend(lines[line_index]) is None:
														snake_case_				=							lines[line_index]
														# If we have everything on a single line, let's deal with it.
														if _re_one_line_import_struct.search(a_):
																					snake_case_				=							_re_one_line_import_struct.search(a_).groups()[0]
																					snake_case_				=							re.findall('\[([^\]]+)\]'							,     a_)
																					for imp in imports:
																												objects.extend([obj[1:-1] for obj in imp.split(', ')])
																					line_index += 1
																					continue
														snake_case_				=							_re_import_struct_key_value.search(a_)
														if single_line_import_search is not None:
																					snake_case_				=							[obj[1:-1] for obj in single_line_import_search.groups()[0].split(', ') if len(a_) > 0]
																					objects.extend(a_)
														elif line.startswith(' ' * 8 + '"'):
																					objects.append(line[9:-3])
														line_index += 1

							snake_case_				=							{'none': objects}
							# Let's continue with backend-specific objects in _import_structure
							while not lines[line_index].startswith('if TYPE_CHECKING'):
														# If the line is an if not is_backend_available, we grab all objects associated.
														snake_case_				=							find_backend(lines[line_index])
														# Check if the backend declaration is inside a try block:
														if _re_try.search(lines[line_index - 1]) is None:
																					snake_case_				=							None

														if backend is not None:
																					line_index += 1

																					# Scroll until we hit the else block of try-except-else
																					while _re_else.search(lines[line_index]) is None:
																												line_index += 1

																					line_index += 1

																					snake_case_				=							[]
																					# Until we unindent, add backend objects to the list
																					while len(lines[line_index]) <= 1 or lines[line_index].startswith(' ' * 4):
																												snake_case_				=							lines[line_index]
																												if _re_import_struct_add_one.search(a_) is not None:
																																			objects.append(_re_import_struct_add_one.search(a_).groups()[0])
																												elif _re_import_struct_add_many.search(a_) is not None:
																																			snake_case_				=							_re_import_struct_add_many.search(a_).groups()[0].split(', ')
																																			snake_case_				=							[obj[1:-1] for obj in imports if len(a_) > 0]
																																			objects.extend(a_)
																												elif _re_between_brackets.search(a_) is not None:
																																			snake_case_				=							_re_between_brackets.search(a_).groups()[0].split(', ')
																																			snake_case_				=							[obj[1:-1] for obj in imports if len(a_) > 0]
																																			objects.extend(a_)
																												elif _re_quote_object.search(a_) is not None:
																																			objects.append(_re_quote_object.search(a_).groups()[0])
																												elif line.startswith(' ' * 8 + '"'):
																																			objects.append(line[9:-3])
																												elif line.startswith(' ' * 12 + '"'):
																																			objects.append(line[13:-3])
																												line_index += 1

																					snake_case_				=							objects
														else:
																					line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
							snake_case_				=							[]
							while (
							    line_index < len(a_)
							    and find_backend(lines[line_index]) is None
							    and not lines[line_index].startswith('else')
							):
														snake_case_				=							lines[line_index]
														snake_case_				=							_re_import.search(a_)
														if single_line_import_search is not None:
																					objects.extend(single_line_import_search.groups()[0].split(', '))
														elif line.startswith(' ' * 8):
																					objects.append(line[8:-2])
														line_index += 1

							snake_case_				=							{'none': objects}
							# Let's continue with backend-specific objects
							while line_index < len(a_):
														# If the line is an if is_backend_available, we grab all objects associated.
														snake_case_				=							find_backend(lines[line_index])
														# Check if the backend declaration is inside a try block:
														if _re_try.search(lines[line_index - 1]) is None:
																					snake_case_				=							None

														if backend is not None:
																					line_index += 1

																					# Scroll until we hit the else block of try-except-else
																					while _re_else.search(lines[line_index]) is None:
																												line_index += 1

																					line_index += 1

																					snake_case_				=							[]
																					# Until we unindent, add backend objects to the list
																					while len(lines[line_index]) <= 1 or lines[line_index].startswith(' ' * 8):
																												snake_case_				=							lines[line_index]
																												snake_case_				=							_re_import.search(a_)
																												if single_line_import_search is not None:
																																			objects.extend(single_line_import_search.groups()[0].split(', '))
																												elif line.startswith(' ' * 12):
																																			objects.append(line[12:-2])
																												line_index += 1

																					snake_case_				=							objects
														else:
																					line_index += 1

							return import_dict_objects, type_hint_objects





def 						__UpperCAmelCase			(     a_							,     a_):

							def find_duplicates(a_):
														return [k for k, v in collections.Counter(a_).items() if v > 1]

							if list(import_dict_objects.keys()) != list(type_hint_objects.keys()):
														return ["Both sides of the init do not have the same backends!"]

							snake_case_				=							[]
							for key in import_dict_objects.keys():
														snake_case_				=							find_duplicates(import_dict_objects[key])
														if duplicate_imports:
																					errors.append(f'''Duplicate _import_structure definitions for: {duplicate_imports}''')
														snake_case_				=							find_duplicates(type_hint_objects[key])
														if duplicate_type_hints:
																					errors.append(f'''Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}''')

														if sorted(set(import_dict_objects[key])) != sorted(set(type_hint_objects[key])):
																					snake_case_				=							'base imports' if key == 'none' else f'''{key} backend'''
																					errors.append(f'''Differences for {name}:''')
																					for a in type_hint_objects[key]:
																												if a not in import_dict_objects[key]:
																																			errors.append(f'''  {a} in TYPE_HINT but not in _import_structure.''')
																					for a in import_dict_objects[key]:
																												if a not in type_hint_objects[key]:
																																			errors.append(f'''  {a} in _import_structure but not in TYPE_HINT.''')
							return errors





def 						__UpperCAmelCase			(     ):
							snake_case_				=							[]
							for root, _, files in os.walk(a_):
														if "__init__.py" in files:
																					snake_case_				=							os.path.join(a_							,     '__init__.py')
																					snake_case_				=							parse_init(a_)
																					if objects is not None:
																												snake_case_				=							analyze_results(*a_)
																												if len(a_) > 0:
																																			snake_case_				=							f'''Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'''
																																			failures.append('\n'.join(a_))
							if len(a_) > 0:
														raise ValueError('\n\n'.join(a_))





def 						__UpperCAmelCase			(     ):
							snake_case_				=							[]
							for path, directories, files in os.walk(a_):
														for folder in directories:
																					# Ignore private modules
																					if folder.startswith('_'):
																												directories.remove(a_)
																												continue
																					# Ignore leftovers from branches (empty folders apart from pycache)
																					if len(list((Path(a_) / folder).glob('*.py'))) == 0:
																												continue
																					snake_case_				=							str((Path(a_) / folder).relative_to(a_))
																					snake_case_				=							short_path.replace(os.path.sep							,     '.')
																					submodules.append(a_)
														for fname in files:
																					if fname == "__init__.py":
																												continue
																					snake_case_				=							str((Path(a_) / fname).relative_to(a_))
																					snake_case_				=							short_path.replace('.py'							,     '').replace(os.path.sep							,     '.')
																					if len(submodule.split('.')) == 1:
																												submodules.append(a_)
							return submodules


lowercase  							=					[
    "convert_pytorch_checkpoint_to_tf2",
    "modeling_flax_pytorch_utils",
]





def 						__UpperCAmelCase			(     ):
							# This is to make sure the transformers module imported is the one in the repo.
							snake_case_				=							importlib.util.spec_from_file_location(
							    'transformers'							,     os.path.join(a_							,     '__init__.py')							,     submodule_search_locations=[PATH_TO_TRANSFORMERS]							,     )
							snake_case_				=							spec.loader.load_module()

							snake_case_				=							[
							    module
							    for module in get_transformers_submodules()
							    if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
							]
							if len(a_) > 0:
														snake_case_				=							'\n'.join(f'''- {module}''' for module in module_not_registered)
														raise ValueError(
														    'The following submodules are not properly registered in the main init of Transformers:\n'
														    f'''{list_of_modules}\n'''
														    'Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.')


if __name__ == "__main__":
	check_all_inits()
	check_submodules()





