


"""simple docstring"""







import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel






class 						_A  :







    """simple docstring"""




    def __init__(					self				:    Dict				, __UpperCAmelCase				:    Tuple				, __UpperCAmelCase				:    Any=13				, __UpperCAmelCase				:    Tuple=7				, __UpperCAmelCase				:    Optional[int]=True				, __UpperCAmelCase				:    int=True				, __UpperCAmelCase				:    Any=True				, __UpperCAmelCase				:    Optional[Any]=True				, __UpperCAmelCase				:    int=99				, __UpperCAmelCase				:    Dict=32				, __UpperCAmelCase				:    Union[str, Any]=5				, __UpperCAmelCase				:    str=4				, __UpperCAmelCase				:    Optional[Any]=4				, __UpperCAmelCase				:    Dict="gelu"				, __UpperCAmelCase				:    Union[str, Any]=0.0				, __UpperCAmelCase				:    Optional[Any]=0.1				, __UpperCAmelCase				:    int=True				, __UpperCAmelCase				:    str=512				, __UpperCAmelCase				:    Any=16				, __UpperCAmelCase				:    Optional[Any]=2				, __UpperCAmelCase				:    int=0.02				, __UpperCAmelCase				:    List[Any]=3				, __UpperCAmelCase				:    Union[str, Any]=4				, __UpperCAmelCase				:    List[str]=None				, ):
      a						:		str      							=			parent
      a						:		Union[str, Any]      							=			batch_size
      a						:		Optional[Any]      							=			seq_length
      a						:		int      							=			is_training
      a						:		int      							=			use_input_mask
      a						:		Optional[Any]      							=			use_token_type_ids
      a						:		Any      							=			use_labels
      a						:		Optional[int]      							=			vocab_size
      a						:		Optional[int]      							=			hidden_size
      a						:		Any      							=			num_hidden_layers
      a						:		str      							=			num_attention_heads
      a						:		int      							=			intermediate_multiple_size
      a						:		Any      							=			hidden_act
      a						:		Union[str, Any]      							=			hidden_dropout
      a						:		int      							=			attention_dropout
      a						:		str      							=			weight_tying
      a						:		Optional[int]      							=			max_position_embeddings
      a						:		Optional[int]      							=			type_vocab_size
      a						:		int      							=			type_sequence_label_size
      a						:		Optional[int]      							=			initializer_range
      a						:		str      							=			num_labels
      a						:		List[Any]      							=			num_choices
      a						:		Any      							=			scope






    def 	__snake_case							(					self				:    List[str]):
      a						:		str      							=			ids_tensor([self.batch_size, self.seq_length]				, self.vocab_size)

      a						:		Union[str, Any]      							=			None
      if self.use_input_mask:
        a						:		Union[str, Any]      							=			random_attention_mask([self.batch_size, self.seq_length])

      a						:		Union[str, Any]      							=			None
      if self.use_labels:
        a						:		Tuple      							=			ids_tensor([self.batch_size, self.seq_length]				, self.num_labels)

      a						:		Any      							=			self.get_config()

      return config, input_ids, input_mask, token_labels






    def 	__snake_case							(					self				:    int):
      return GPTNeoXJapaneseConfig(
          vocab_size=self.vocab_size				, hidden_size=self.hidden_size				, num_hidden_layers=self.num_hidden_layers				, num_attention_heads=self.num_attention_heads				, intermediate_multiple_size=self.intermediate_multiple_size				, hidden_act=self.hidden_act				, hidden_dropout=self.hidden_dropout				, attention_dropout=self.attention_dropout				, weight_tying=self.weight_tying				, max_position_embeddings=self.max_position_embeddings				, type_vocab_size=self.type_vocab_size				, is_decoder=__UpperCAmelCase				, initializer_range=self.initializer_range				, )






    def 	__snake_case							(					self				:    Union[str, Any]):
      a							,		a							,		a							,		a						:		Tuple      							=			self.prepare_config_and_inputs()

      a						:		str      							=			True

      return config, input_ids, input_mask, token_labels






    def 	__snake_case							(					self				:    Any				, __UpperCAmelCase				:    Any				, __UpperCAmelCase				:    Union[str, Any]				, __UpperCAmelCase				:    List[str]):
      a						:		str      							=			GPTNeoXJapaneseModel(config=__UpperCAmelCase)
      model.to(__UpperCAmelCase)
      model.eval()
      a						:		str      							=			model(__UpperCAmelCase				, attention_mask=__UpperCAmelCase)
      a						:		Any      							=			model(__UpperCAmelCase)
      self.parent.assertEqual(result.last_hidden_state.shape				, (self.batch_size, self.seq_length, self.hidden_size))






    def 	__snake_case							(					self				:    Optional[Any]				, __UpperCAmelCase				:    Dict				, __UpperCAmelCase				:    Tuple				, __UpperCAmelCase				:    str):
      a						:		Optional[Any]      							=			True
      a						:		Dict      							=			GPTNeoXJapaneseModel(__UpperCAmelCase)
      model.to(__UpperCAmelCase)
      model.eval()
      a						:		Tuple      							=			model(__UpperCAmelCase				, attention_mask=__UpperCAmelCase)
      self.parent.assertEqual(result.last_hidden_state.shape				, (self.batch_size, self.seq_length, self.hidden_size))






    def 	__snake_case							(					self				:    Union[str, Any]				, __UpperCAmelCase				:    str				, __UpperCAmelCase				:    Union[str, Any]				, __UpperCAmelCase				:    Any				, __UpperCAmelCase				:    Optional[int]):
      a						:		List[Any]      							=			GPTNeoXJapaneseForCausalLM(config=__UpperCAmelCase)
      model.to(__UpperCAmelCase)
      model.eval()
      a						:		List[Any]      							=			model(__UpperCAmelCase				, attention_mask=__UpperCAmelCase				, labels=__UpperCAmelCase)
      self.parent.assertEqual(result.logits.shape				, (self.batch_size, self.seq_length, self.vocab_size))






    def 	__snake_case							(					self				:    List[Any]				, __UpperCAmelCase				:    Union[str, Any]				, __UpperCAmelCase				:    Tuple				, __UpperCAmelCase				:    Union[str, Any]):
      a						:		List[Any]      							=			True
      a						:		Any      							=			GPTNeoXJapaneseForCausalLM(config=__UpperCAmelCase)
      model.to(__UpperCAmelCase)
      model.eval()

      # first forward pass
      a						:		Optional[Any]      							=			model(__UpperCAmelCase				, attention_mask=__UpperCAmelCase				, use_cache=__UpperCAmelCase)
      a						:		Optional[Any]      							=			outputs.past_key_values

      # create hypothetical multiple next token and extent to next_input_ids
      a						:		Union[str, Any]      							=			ids_tensor((self.batch_size, 3)				, config.vocab_size)
      a						:		Tuple      							=			ids_tensor((self.batch_size, 3)				, vocab_size=2)

      # append to next input_ids and
      a						:		List[Any]      							=			torch.cat([input_ids, next_tokens]				, dim=-1)
      a						:		Optional[int]      							=			torch.cat([input_mask, next_mask]				, dim=-1)

      a						:		Optional[Any]      							=			model(__UpperCAmelCase				, attention_mask=__UpperCAmelCase				, output_hidden_states=__UpperCAmelCase)
      a						:		List[Any]      							=			output_from_no_past["hidden_states"][0]
      a						:		Tuple      							=			model(
          __UpperCAmelCase				, attention_mask=__UpperCAmelCase				, past_key_values=__UpperCAmelCase				, output_hidden_states=__UpperCAmelCase				, )["hidden_states"][0]

      # select random slice
      a						:		List[Any]      							=			ids_tensor((1,)				, output_from_past.shape[-1]).item()
      a						:		List[str]      							=			output_from_no_past[:, -3:, random_slice_idx].detach()
      a						:		List[Any]      							=			output_from_past[:, :, random_slice_idx].detach()

      self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

      # test that outputs are equal for slice
      self.parent.assertTrue(torch.allclose(__UpperCAmelCase				, __UpperCAmelCase				, atol=1e-3))






    def 	__snake_case							(					self				:    Optional[int]):
      a						:		Tuple      							=			self.prepare_config_and_inputs()
      a							,		a							,		a							,		a						:		Any      							=			config_and_inputs
      a						:		int      							=			{"input_ids": input_ids, "attention_mask": input_mask}
      return config, inputs_dict






@require_torch
class 						_A  (  _a    ,_a    ,unittest.TestCase  ):







    """simple docstring"""




    UpperCAmelCase       :							int	=       (GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
    UpperCAmelCase       :							List[Any]	=       (GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
    UpperCAmelCase       :							List[Any]	=       (
        {"""feature-extraction""": GPTNeoXJapaneseModel, """text-generation""": GPTNeoXJapaneseForCausalLM}
        if is_torch_available()
        else {}
    )
    UpperCAmelCase       :							str	=       False
    UpperCAmelCase       :							Optional[int]	=       False
    UpperCAmelCase       :							Optional[int]	=       False
    UpperCAmelCase       :							Dict	=       False






    def 	__snake_case							(					self				:    Any):
      a						:		Optional[int]      							=			GPTNeoXJapaneseModelTester(self)
      a						:		Optional[int]      							=			ConfigTester(self				, config_class=__UpperCAmelCase				, hidden_size=37)






    def 	__snake_case							(					self				:    List[str]):
      self.config_tester.run_common_tests()






    def 	__snake_case							(					self				:    Tuple):
      a							,		a							,		a							,		a						:		str      							=			self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_model(__UpperCAmelCase				, __UpperCAmelCase				, __UpperCAmelCase)






    def 	__snake_case							(					self				:    Dict):
      a							,		a							,		a							,		a						:		Tuple      							=			self.model_tester.prepare_config_and_inputs_for_decoder()
      self.model_tester.create_and_check_model_as_decoder(__UpperCAmelCase				, __UpperCAmelCase				, __UpperCAmelCase)






    def 	__snake_case							(					self				:    Optional[int]):
      # This regression test was failing with PyTorch < 1.3
      a							,		a							,		a							,		a						:		int      							=			self.model_tester.prepare_config_and_inputs_for_decoder()

      a						:		Union[str, Any]      							=			None

      self.model_tester.create_and_check_model_as_decoder(__UpperCAmelCase				, __UpperCAmelCase				, __UpperCAmelCase)






    def 	__snake_case							(					self				:    str):
      a							,		a							,		a							,		a						:		int      							=			self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_decoder_model_past_large_inputs(__UpperCAmelCase				, __UpperCAmelCase				, __UpperCAmelCase)






    def 	__snake_case							(					self				:    Optional[int]):
      a						:		Optional[Any]      							=			self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_for_causal_lm(*__UpperCAmelCase)






    @slow
    def 	__snake_case							(					self				:    List[str]):
      a						:		Optional[int]      							=			"abeja/gpt-neox-japanese-2.7b"

      a						:		int      							=			["データサイエンティストとは、", "100年後に必要とされる会社は、", "フルリモートの環境で働くために必要なことは、", "国境の長いトンネルを抜けると", "美味しい日本食といえば、"]

      a						:		Union[str, Any]      							=			[
          "データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。",
          "100年後に必要とされる会社は、「人」が中心の会社です。",
          "フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。",
          "国境の長いトンネルを抜けると、そこは雪国だった。",
          "美味しい日本食といえば、やっぱりお寿司ですよね。",
      ]

      a						:		Tuple      							=			GPTNeoXJapaneseTokenizer.from_pretrained(__UpperCAmelCase)
      a						:		List[Any]      							=			GPTNeoXJapaneseForCausalLM.from_pretrained(__UpperCAmelCase)

      a						:		int      							=			[]
      for prompt in prompts:
        a						:		List[str]      							=			tokenizer(__UpperCAmelCase				, return_tensors="pt").input_ids
        a						:		List[str]      							=			model.generate(__UpperCAmelCase				, max_length=50)
        a						:		Union[str, Any]      							=			tokenizer.batch_decode(__UpperCAmelCase				, skip_special_tokens=__UpperCAmelCase)
        predicted_outputs += generated_string
      self.assertListEqual(__UpperCAmelCase				, __UpperCAmelCase)








import logging
import os
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


lowercase__       ={
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING,
    'error': logging.ERROR,
    'critical': logging.CRITICAL,
}

lowercase__       =logging.WARNING




def    __UpperCamelCase     (				):
 __a :  Optional[Any]						=  os.getenv('''DATASETS_VERBOSITY'''  ,   lowerCAmelCase__  )
 if env_level_str:
  if env_level_str in log_levels:
   return log_levels[env_level_str]
  else:
   logging.getLogger().warning(
       f"Unknown option DATASETS_VERBOSITY={env_level_str}, "
       f"has to be one of: { ', '.join(log_levels.keys()  ) }"  )
 return _default_log_level




def    __UpperCamelCase     (				):
 return __name__.split('''.'''  )[0]




def    __UpperCamelCase     (				):
 return logging.getLogger(_get_library_name()  )




def    __UpperCamelCase     (				):
 # Apply our default configuration to the library root logger.
 __a :  str						=  _get_library_root_logger()
 library_root_logger.setLevel(_get_default_logging_level()  )




def    __UpperCamelCase     (				):
 __a :  Any						=  _get_library_root_logger()
 library_root_logger.setLevel(logging.NOTSET  )




def    __UpperCamelCase     (				lowerCAmelCase__							:				Optional[str] = None  ):
 if name is None:
  __a :  Union[str, Any]						=  _get_library_name()
 return logging.getLogger(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return _get_library_root_logger().getEffectiveLevel()




def    __UpperCamelCase     (				lowerCAmelCase__							:				int  ):
 _get_library_root_logger().setLevel(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return set_verbosity(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return set_verbosity(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return set_verbosity(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return set_verbosity(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 __a :  Union[str, Any]						=  False




def    __UpperCamelCase     (				):
 __a :  Tuple						=  True


# Configure the library root logger at the module level (singleton-like)
_configure_library_root_logger()



class 		UpperCamelCase__  :
   def __init__(self :					str					,				*snake_case_ :					str					,				**snake_case_ :					Union[str, Any]   ):  # pylint: disable=unused-argument
    __a :  Optional[Any]						=  args[0] if args else None
   def __iter__(self :					List[str]   ):
    return iter(self._iterator   )
   def __getattr__(self :					str					,				snake_case_ :					Optional[Any]   ):

    def empty_fn(*snake_case_ :					int					,				**snake_case_ :					int   ):  # pylint: disable=unused-argument
     return

    return empty_fn
   def __enter__(self :					Union[str, Any]   ):
    return self







   def __exit__(self :					str					,				snake_case_ :					List[str]					,				snake_case_ :					int					,				snake_case_ :					Optional[Any]   ):
    return





lowercase__       =True



class 		UpperCamelCase__  :
   def __call__(self :					Tuple					,				*snake_case_ :					str					,				snake_case_ :					str=False					,				**snake_case_ :					Dict   ):
    if _tqdm_active and not disable:
     return tqdm_lib.tqdm(*snake_case_					,				**snake_case_   )
    else:
     return EmptyTqdm(*snake_case_					,				**snake_case_   )
   def       lowerCAmelCase			(self :					Optional[Any]					,				*snake_case_ :					Union[str, Any]					,				**snake_case_ :					Optional[Any]   ):
    __a :  List[Any]						=  None
    if _tqdm_active:
     return tqdm_lib.tqdm.set_lock(*snake_case_					,				**snake_case_   )







   def       lowerCAmelCase			(self :					str   ):
    if _tqdm_active:
     return tqdm_lib.tqdm.get_lock()





lowercase__       =_tqdm_cls()




def    __UpperCamelCase     (				):
 global _tqdm_active
 return bool(_tqdm_active  )




def    __UpperCamelCase     (				):
 global _tqdm_active
 __a :  Dict						=  True




def    __UpperCamelCase     (				):
 global _tqdm_active
 __a :  Union[str, Any]						=  False
