






import unittest

from transformers import BertGenerationTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowercase  							=					"▁"

lowercase  							=					get_tests_dir("fixtures/test_sentencepiece.model")




@require_sentencepiece
class  UpperCamelCase_      (			snake_case_       ,					unittest.TestCase				):



	'''simple docstring'''




	lowerCAmelCase													= BertGenerationTokenizer
	lowerCAmelCase													= False
	lowerCAmelCase													= True




	def 							_UpperCamelCase			(       self					)       ->    List[str]:
								super().setUp()

								snake_case_				=							BertGenerationTokenizer(a       ,			keep_accents=a					)
								tokenizer.save_pretrained(self.tmpdirname					)




	def 							_UpperCamelCase			(       self					)       ->    Tuple:
								snake_case_				=							'<s>'
								snake_case_				=							1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(a					)       ,			a					)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(a					)       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    Dict:
								snake_case_				=							list(self.get_tokenizer().get_vocab().keys()					)

								self.assertEqual(vocab_keys[0]       ,			'<unk>'					)
								self.assertEqual(vocab_keys[1]       ,			'<s>'					)
								self.assertEqual(vocab_keys[-1]       ,			'<pad>'					)
								self.assertEqual(len(a					)       ,			10_02					)




	def 							_UpperCamelCase			(       self					)       ->    Union[str, Any]:
								self.assertEqual(self.get_tokenizer().vocab_size       ,			10_00					)




	def 							_UpperCamelCase			(       self					)       ->    Optional[int]:
								snake_case_				=							BertGenerationTokenizer(a       ,			keep_accents=a					)

								snake_case_				=							tokenizer.tokenize('This is a test'					)
								self.assertListEqual(a       ,			['▁This', '▁is', '▁a', '▁t', 'est']					)

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(a					)       ,			[2_85, 46, 10, 1_70, 3_82]       ,			)

								snake_case_				=							tokenizer.tokenize('I was born in 92000, and this is falsé.'					)
								self.assertListEqual(
								    a       ,			[
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '9',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        'é',
								        '.',
								    ]       ,			)
								snake_case_				=							tokenizer.convert_tokens_to_ids(a					)
								self.assertListEqual(
								    a       ,			[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]       ,			)

								snake_case_				=							tokenizer.convert_ids_to_tokens(a					)
								self.assertListEqual(
								    a       ,			[
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '<unk>',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        '<unk>',
								        '.',
								    ]       ,			)




	@cached_property
	def 							_UpperCamelCase			(       self					)       ->    Optional[int]:
								return BertGenerationTokenizer.from_pretrained('google/bert_for_seq_generation_L-24_bbc_encoder'					)




	@slow
	def 							_UpperCamelCase			(       self					)       ->    Optional[Any]:
								snake_case_				=							'Hello World!'
								snake_case_				=							[1_85_36, 22_60, 1_01]

								self.assertListEqual(a       ,			self.big_tokenizer.encode(a					)					)




	@slow
	def 							_UpperCamelCase			(       self					)       ->    int:
								snake_case_				=							(
								    'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
								    ' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'
								)
								snake_case_				=							[
								    8_71,
								    4_19,
								    3_58,
								    9_46,
								    9_91,
								    25_21,
								    4_52,
								    3_58,
								    13_57,
								    3_87,
								    77_51,
								    35_36,
								    1_12,
								    9_85,
								    4_56,
								    1_26,
								    8_65,
								    9_38,
								    54_00,
								    57_34,
								    4_58,
								    13_68,
								    4_67,
								    7_86,
								    24_62,
								    52_46,
								    11_59,
								    6_33,
								    8_65,
								    45_19,
								    4_57,
								    5_82,
								    8_52,
								    25_57,
								    4_27,
								    9_16,
								    5_08,
								    4_05,
								    3_43_24,
								    4_97,
								    3_91,
								    4_08,
								    1_13_42,
								    12_44,
								    3_85,
								    1_00,
								    9_38,
								    9_85,
								    4_56,
								    5_74,
								    3_62,
								    1_25_97,
								    32_00,
								    31_29,
								    11_72,
								]

								self.assertListEqual(a       ,			self.big_tokenizer.encode(a					)					)




	@require_torch
	@slow
	def 							_UpperCamelCase			(       self					)       ->    List[str]:
								import torch

								from transformers import BertGenerationConfig, BertGenerationEncoder

								# Build sequence
								snake_case_				=							list(self.big_tokenizer.get_vocab().keys()					)[:10]
								snake_case_				=							' '.join(a					)
								snake_case_				=							self.big_tokenizer.encode_plus(a       ,			return_tensors='pt'       ,			return_token_type_ids=a					)
								snake_case_				=							self.big_tokenizer.batch_encode_plus(
								    [sequence + ' ' + sequence]       ,			return_tensors='pt'       ,			return_token_type_ids=a					)

								snake_case_				=							BertGenerationConfig()
								snake_case_				=							BertGenerationEncoder(a					)

								assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

								with torch.no_grad():
															model(**a					)
															model(**a					)






	@slow
	def 							_UpperCamelCase			(       self					)       ->    Union[str, Any]:
								# fmt: off
								snake_case_				=							{'input_ids': [[3_92_86, 4_58, 3_63_35, 20_01, 4_56, 1_30_73, 1_32_66, 4_55, 1_13, 77_46, 17_41, 1_11_57, 3_91, 1_30_73, 1_32_66, 4_55, 1_13, 39_67, 3_54_12, 1_13, 49_36, 1_09, 38_70, 23_77, 1_13, 3_00_84, 4_57_20, 4_58, 1_34, 1_74_96, 1_12, 5_03, 1_16_72, 1_13, 1_18, 1_12, 56_65, 1_33_47, 3_86_87, 1_12, 14_96, 3_13_89, 1_12, 32_68, 4_72_64, 1_34, 9_62, 1_12, 1_63_77, 80_35, 2_31_30, 4_30, 1_21_69, 1_55_18, 2_85_92, 4_58, 1_46, 4_16_97, 1_09, 3_91, 1_21_69, 1_55_18, 1_66_89, 4_58, 1_46, 4_13_58, 1_09, 4_52, 7_26, 40_34, 1_11, 7_63, 3_54_12, 50_82, 3_88, 19_03, 1_11, 90_51, 3_91, 28_70, 4_89_18, 19_00, 11_23, 5_50, 9_98, 1_12, 95_86, 1_59_85, 4_55, 3_91, 4_10, 2_29_55, 3_76_36, 1_14], [4_48, 1_74_96, 4_19, 36_63, 3_85, 7_63, 1_13, 2_75_33, 28_70, 32_83, 1_30_43, 16_39, 2_47_13, 5_23, 6_56, 2_40_13, 1_85_50, 25_21, 5_17, 2_70_14, 2_12_44, 4_20, 12_12, 14_65, 3_91, 9_27, 48_33, 3_88, 5_78, 1_17_86, 1_14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [4_84, 21_69, 76_87, 2_19_32, 1_81_46, 7_26, 3_63, 1_70_32, 33_91, 1_14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=a       ,			model_name='google/bert_for_seq_generation_L-24_bbc_encoder'       ,			revision='c817d1fd1be2ffa69431227a1fe320544943d4db'       ,			)













import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel




class  UpperCamelCase_      :



	'''simple docstring'''




	def __init__(       self       ,			a       ,			a=13       ,			a=7       ,			a=True       ,			a=True       ,			a=True       ,			a=True       ,			a=99       ,			a=32       ,			a=5       ,			a=4       ,			a=4       ,			a="gelu"       ,			a=0.0       ,			a=0.1       ,			a=True       ,			a=5_12       ,			a=16       ,			a=2       ,			a=0.02       ,			a=3       ,			a=4       ,			a=None       ,			)       ->    Optional[Any]:
								snake_case_				=							parent
								snake_case_				=							batch_size
								snake_case_				=							seq_length
								snake_case_				=							is_training
								snake_case_				=							use_input_mask
								snake_case_				=							use_token_type_ids
								snake_case_				=							use_labels
								snake_case_				=							vocab_size
								snake_case_				=							hidden_size
								snake_case_				=							num_hidden_layers
								snake_case_				=							num_attention_heads
								snake_case_				=							intermediate_multiple_size
								snake_case_				=							hidden_act
								snake_case_				=							hidden_dropout
								snake_case_				=							attention_dropout
								snake_case_				=							weight_tying
								snake_case_				=							max_position_embeddings
								snake_case_				=							type_vocab_size
								snake_case_				=							type_sequence_label_size
								snake_case_				=							initializer_range
								snake_case_				=							num_labels
								snake_case_				=							num_choices
								snake_case_				=							scope




	def 							_UpperCamelCase			(       self					)       ->    List[Any]:
								snake_case_				=							ids_tensor([self.batch_size, self.seq_length]       ,			self.vocab_size					)

								snake_case_				=							None
								if self.use_input_mask:
															snake_case_				=							random_attention_mask([self.batch_size, self.seq_length]					)

								snake_case_				=							None
								if self.use_labels:
															snake_case_				=							ids_tensor([self.batch_size, self.seq_length]       ,			self.num_labels					)

								snake_case_				=							self.get_config()

								return config, input_ids, input_mask, token_labels




	def 							_UpperCamelCase			(       self					)       ->    Dict:
								return GPTNeoXJapaneseConfig(
								    vocab_size=self.vocab_size       ,			hidden_size=self.hidden_size       ,			num_hidden_layers=self.num_hidden_layers       ,			num_attention_heads=self.num_attention_heads       ,			intermediate_multiple_size=self.intermediate_multiple_size       ,			hidden_act=self.hidden_act       ,			hidden_dropout=self.hidden_dropout       ,			attention_dropout=self.attention_dropout       ,			weight_tying=self.weight_tying       ,			max_position_embeddings=self.max_position_embeddings       ,			type_vocab_size=self.type_vocab_size       ,			is_decoder=a       ,			initializer_range=self.initializer_range       ,			)




	def 							_UpperCamelCase			(       self					)       ->    int:
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.prepare_config_and_inputs()

								snake_case_				=							True

								return config, input_ids, input_mask, token_labels




	def 							_UpperCamelCase			(       self       ,			a       ,			a       ,			a					)       ->    Any:
								snake_case_				=							GPTNeoXJapaneseModel(config=a					)
								model.to(a					)
								model.eval()
								snake_case_				=							model(a       ,			attention_mask=a					)
								snake_case_				=							model(a					)
								self.parent.assertEqual(result.last_hidden_state.shape       ,			(self.batch_size, self.seq_length, self.hidden_size)					)




	def 							_UpperCamelCase			(       self       ,			a       ,			a       ,			a					)       ->    Union[str, Any]:
								snake_case_				=							True
								snake_case_				=							GPTNeoXJapaneseModel(a					)
								model.to(a					)
								model.eval()
								snake_case_				=							model(a       ,			attention_mask=a					)
								self.parent.assertEqual(result.last_hidden_state.shape       ,			(self.batch_size, self.seq_length, self.hidden_size)					)




	def 							_UpperCamelCase			(       self       ,			a       ,			a       ,			a       ,			a					)       ->    int:
								snake_case_				=							GPTNeoXJapaneseForCausalLM(config=a					)
								model.to(a					)
								model.eval()
								snake_case_				=							model(a       ,			attention_mask=a       ,			labels=a					)
								self.parent.assertEqual(result.logits.shape       ,			(self.batch_size, self.seq_length, self.vocab_size)					)




	def 							_UpperCamelCase			(       self       ,			a       ,			a       ,			a					)       ->    Tuple:
								snake_case_				=							True
								snake_case_				=							GPTNeoXJapaneseForCausalLM(config=a					)
								model.to(a					)
								model.eval()

								# first forward pass
								snake_case_				=							model(a       ,			attention_mask=a       ,			use_cache=a					)
								snake_case_				=							outputs.past_key_values

								# create hypothetical multiple next token and extent to next_input_ids
								snake_case_				=							ids_tensor((self.batch_size, 3)       ,			config.vocab_size					)
								snake_case_				=							ids_tensor((self.batch_size, 3)       ,			vocab_size=2					)

								# append to next input_ids and
								snake_case_				=							torch.cat([input_ids, next_tokens]       ,			dim=-1					)
								snake_case_				=							torch.cat([input_mask, next_mask]       ,			dim=-1					)

								snake_case_				=							model(a       ,			attention_mask=a       ,			output_hidden_states=a					)
								snake_case_				=							output_from_no_past['hidden_states'][0]
								snake_case_				=							model(
								    a       ,			attention_mask=a       ,			past_key_values=a       ,			output_hidden_states=a       ,			)['hidden_states'][0]

								# select random slice
								snake_case_				=							ids_tensor((1,)       ,			output_from_past.shape[-1]					).item()
								snake_case_				=							output_from_no_past[:, -3:, random_slice_idx].detach()
								snake_case_				=							output_from_past[:, :, random_slice_idx].detach()

								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]					)

								# test that outputs are equal for slice
								self.parent.assertTrue(torch.allclose(a       ,			a       ,			atol=1E-3					)					)






	def 							_UpperCamelCase			(       self					)       ->    Dict:
								snake_case_				=							self.prepare_config_and_inputs()
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							config_and_inputs
								snake_case_				=							{'input_ids': input_ids, 'attention_mask': input_mask}
								return config, inputs_dict






@require_torch
class  UpperCamelCase_      (			snake_case_       ,					snake_case_       ,					unittest.TestCase				):



	'''simple docstring'''




	lowerCAmelCase													= (GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
	lowerCAmelCase													= (GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
	lowerCAmelCase													= (
	    {'''feature-extraction''': GPTNeoXJapaneseModel, '''text-generation''': GPTNeoXJapaneseForCausalLM}
	    if is_torch_available()
	    else {}
	)
	lowerCAmelCase													= False
	lowerCAmelCase													= False
	lowerCAmelCase													= False
	lowerCAmelCase													= False




	def 							_UpperCamelCase			(       self					)       ->    List[Any]:
								snake_case_				=							GPTNeoXJapaneseModelTester(self					)
								snake_case_				=							ConfigTester(self       ,			config_class=a       ,			hidden_size=37					)




	def 							_UpperCamelCase			(       self					)       ->    str:
								self.config_tester.run_common_tests()




	def 							_UpperCamelCase			(       self					)       ->    Optional[Any]:
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(a       ,			a       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    Union[str, Any]:
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.model_tester.prepare_config_and_inputs_for_decoder()
								self.model_tester.create_and_check_model_as_decoder(a       ,			a       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    Optional[int]:
								# This regression test was failing with PyTorch < 1.3
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.model_tester.prepare_config_and_inputs_for_decoder()

								snake_case_				=							None

								self.model_tester.create_and_check_model_as_decoder(a       ,			a       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    Dict:
								snake_case_			,		snake_case_			,		snake_case_			,		snake_case_				=							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_decoder_model_past_large_inputs(a       ,			a       ,			a					)




	def 							_UpperCamelCase			(       self					)       ->    List[Any]:
								snake_case_				=							self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_causal_lm(*a					)






	@slow
	def 							_UpperCamelCase			(       self					)       ->    Any:
								snake_case_				=							'abeja/gpt-neox-japanese-2.7b'

								snake_case_				=							['データサイエンティストとは、', '100年後に必要とされる会社は、', 'フルリモートの環境で働くために必要なことは、', '国境の長いトンネルを抜けると', '美味しい日本食といえば、']

								snake_case_				=							[
								    'データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。',
								    '100年後に必要とされる会社は、「人」が中心の会社です。',
								    'フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。',
								    '国境の長いトンネルを抜けると、そこは雪国だった。',
								    '美味しい日本食といえば、やっぱりお寿司ですよね。',
								]

								snake_case_				=							GPTNeoXJapaneseTokenizer.from_pretrained(a					)
								snake_case_				=							GPTNeoXJapaneseForCausalLM.from_pretrained(a					)

								snake_case_				=							[]
								for prompt in prompts:
															snake_case_				=							tokenizer(a       ,			return_tensors='pt'					).input_ids
															snake_case_				=							model.generate(a       ,			max_length=50					)
															snake_case_				=							tokenizer.batch_decode(a       ,			skip_special_tokens=a					)
															predicted_outputs += generated_string
								self.assertListEqual(a       ,			a					)





