
import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 		UpperCamelCase__  (       __lowercase    ,unittest.TestCase    ):
   _SCREAMING_SNAKE_CASE      :							Optional[Any]	       =      GPTSanJapaneseTokenizer
   _SCREAMING_SNAKE_CASE      :							Optional[int]	       =      False
   _SCREAMING_SNAKE_CASE      :							Optional[int]	       =      {"do_clean_text": False, "add_prefix_space": False}
   def       lowerCAmelCase			(self :					str   ):
    super().setUp()

    # fmt: off
    __a :  int						=  ['''こん''', '''こんに''', '''にちは''', '''ばんは''', '''世界,㔺界''', '''、''', '''。''', '''<BR>''', '''<SP>''', '''<TAB>''', '''<URL>''', '''<EMAIL>''', '''<TEL>''', '''<DATE>''', '''<PRICE>''', '''<BLOCK>''', '''<KIGOU>''', '''<U2000U2BFF>''', '''<|emoji1|>''', '''<unk>''', '''<|bagoftoken|>''', '''<|endoftext|>''']
    # fmt: on
    __a :  Any						=  {'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
    __a :  Dict						=  {'''unk_token''': '''<unk>'''}

    __a :  Any						=  os.path.join(self.tmpdirname					,				VOCAB_FILES_NAMES['''vocab_file''']   )
    __a :  Dict						=  os.path.join(self.tmpdirname					,				VOCAB_FILES_NAMES['''emoji_file''']   )
    with open(self.vocab_file					,				'''w'''					,				encoding='''utf-8'''   ) as vocab_writer:
     vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]   )   )
    with open(self.emoji_file					,				'''w'''   ) as emoji_writer:
     emoji_writer.write(json.dumps(snake_case_   )   )
   def       lowerCAmelCase			(self :					Dict					,				**snake_case_ :					Optional[int]   ):
    kwargs.update(self.special_tokens_map   )
    return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname					,				**snake_case_   )
   def       lowerCAmelCase			(self :					int					,				snake_case_ :					List[Any]   ):
    __a :  int						=  '''こんにちは、世界。 \nこんばんは、㔺界。😀'''
    __a :  Tuple						=  '''こんにちは、世界。 \nこんばんは、世界。😀'''
    return input_text, output_text
   def       lowerCAmelCase			(self :					int					,				snake_case_ :					Optional[int]   ):
    __a				,       __a :  Dict						=  self.get_input_output_texts(snake_case_   )
    __a :  List[Any]						=  tokenizer.encode(snake_case_					,				add_special_tokens=snake_case_   )
    __a :  Any						=  tokenizer.decode(snake_case_					,				clean_up_tokenization_spaces=snake_case_   )
    return text, ids
   def       lowerCAmelCase			(self :					Dict   ):
    pass  # TODO add if relevant
   def       lowerCAmelCase			(self :					Any   ):
    pass  # TODO add if relevant
   def       lowerCAmelCase			(self :					List[str]   ):
    pass  # TODO add if relevant
   def       lowerCAmelCase			(self :					str   ):
    __a :  Optional[Any]						=  self.get_tokenizer()

    # Testing tokenization
    __a :  Tuple						=  '''こんにちは、世界。　こんばんは、㔺界。'''
    __a :  str						=  ['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
    __a :  Union[str, Any]						=  tokenizer.tokenize(snake_case_   )
    self.assertListEqual(snake_case_					,				snake_case_   )

    # Testing conversion to ids without special tokens
    __a :  int						=  [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
    __a :  int						=  tokenizer.convert_tokens_to_ids(snake_case_   )
    self.assertListEqual(snake_case_					,				snake_case_   )

    # Testing conversion to ids with special tokens
    __a :  Any						=  tokens + [tokenizer.unk_token]
    __a :  str						=  [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
    __a :  int						=  tokenizer.convert_tokens_to_ids(snake_case_   )
    self.assertListEqual(snake_case_					,				snake_case_   )
   def       lowerCAmelCase			(self :					Dict   ):
    __a :  Tuple						=  self.get_tokenizer()

    # Testing tokenization
    __a :  Dict						=  '''こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。'''
    __a :  List[str]						=  '''こんにちは、、、、世界。こんばんは、、、、世界。'''
    __a :  List[str]						=  tokenizer.encode(snake_case_   )
    __a :  str						=  tokenizer.decode(snake_case_   )
    self.assertEqual(snake_case_					,				snake_case_   )
   @slow
   def       lowerCAmelCase			(self :					Tuple   ):
    __a :  str						=  self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''   )

    # Testing tokenization
    __a :  str						=  '''こんにちは、世界。'''
    __a :  Tuple						=  '''こんばんは、㔺界。😀'''
    __a :  Optional[Any]						=  '''こんにちは、世界。こんばんは、世界。😀'''
    __a :  Union[str, Any]						=  tokenizer.encode(prefix_text + input_text   )
    __a :  str						=  tokenizer.encode(''''''					,				prefix_text=prefix_text + input_text   )
    __a :  str						=  tokenizer.encode(snake_case_					,				prefix_text=snake_case_   )
    __a :  Any						=  tokenizer.decode(snake_case_   )
    __a :  Any						=  tokenizer.decode(snake_case_   )
    __a :  Dict						=  tokenizer.decode(snake_case_   )
    self.assertEqual(snake_case_					,				snake_case_   )
    self.assertEqual(snake_case_					,				snake_case_   )
    self.assertEqual(snake_case_					,				snake_case_   )
   @slow
   def       lowerCAmelCase			(self :					Union[str, Any]   ):
    __a :  Optional[int]						=  self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''   )

    # Testing tokenization
    __a :  Optional[Any]						=  '''こんにちは、世界。'''
    __a :  Optional[int]						=  '''こんばんは、㔺界。😀'''

    __a :  str						=  len(tokenizer.encode(snake_case_   )   ) - 2
    __a :  str						=  len(tokenizer.encode(snake_case_   )   ) - 2

    __a :  int						=  [1] + [0] * (len_prefix + len_text + 1)
    __a :  str						=  [1] * (len_prefix + len_text + 1) + [0]
    __a :  List[str]						=  [1] + [1] * (len_prefix) + [0] * (len_text + 1)

    __a :  Any						=  tokenizer(prefix_text + input_text   ).token_type_ids
    __a :  int						=  tokenizer(''''''					,				prefix_text=prefix_text + input_text   ).token_type_ids
    __a :  List[str]						=  tokenizer(snake_case_					,				prefix_text=snake_case_   ).token_type_ids
    self.assertListEqual(snake_case_					,				snake_case_   )
    self.assertListEqual(snake_case_					,				snake_case_   )
    self.assertListEqual(snake_case_					,				snake_case_   )
   @slow
   def       lowerCAmelCase			(self :					Dict   ):
    __a :  List[Any]						=  self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''   )

    __a :  List[Any]						=  tokenizer.encode('''あンいワ'''   )
    __a :  Optional[int]						=  tokenizer.encode(''''''					,				prefix_text='''あンいワ'''   )
    __a :  Optional[Any]						=  tokenizer.encode('''いワ'''					,				prefix_text='''あン'''   )

    self.assertEqual(tokenizer.decode(snake_case_   )					,				tokenizer.decode(snake_case_   )   )
    self.assertEqual(tokenizer.decode(snake_case_   )					,				tokenizer.decode(snake_case_   )   )
    self.assertNotEqual(snake_case_					,				snake_case_   )
    self.assertNotEqual(snake_case_					,				snake_case_   )
    self.assertEqual(x_token_a[1]					,				x_token_a[-1]   )  # SEG token
    self.assertEqual(x_token_a[1]					,				x_token_a[3]   )  # SEG token
   @slow
   def       lowerCAmelCase			(self :					List[str]   ):
    __a :  List[str]						=  self.tokenizer_class.from_pretrained('''Tanrei/GPTSAN-japanese'''   )

    __a :  Dict						=  [['''武田信玄''', '''は、'''], ['''織田信長''', '''の配下の、''']]
    __a :  str						=  tokenizer(snake_case_					,				padding=snake_case_   )
    __a :  str						=  tokenizer.batch_encode_plus(snake_case_					,				padding=snake_case_   )

    # fmt: off
    __a :  Optional[int]						=  [[3_5_9_9_3, 8_6_4_0, 2_5_9_4_8, 3_5_9_9_8, 3_0_6_4_7, 3_5_6_7_5, 3_5_9_9_9, 3_5_9_9_9], [3_5_9_9_3, 1_0_3_8_2, 9_8_6_8, 3_5_9_9_8, 3_0_6_4_6, 9_4_5_9, 3_0_6_4_6, 3_5_6_7_5]]
    __a :  int						=  [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
    __a :  Tuple						=  [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
    # fmt: on
    self.assertListEqual(x_token.input_ids					,				snake_case_   )
    self.assertListEqual(x_token.token_type_ids					,				snake_case_   )
    self.assertListEqual(x_token.attention_mask					,				snake_case_   )
    self.assertListEqual(x_token_a.input_ids					,				snake_case_   )
    self.assertListEqual(x_token_a.token_type_ids					,				snake_case_   )
    self.assertListEqual(x_token_a.attention_mask					,				snake_case_   )
   def       lowerCAmelCase			(self :					Tuple   ):
    # Intentionally convert some words to accommodate character fluctuations unique to Japanese
    pass







   def       lowerCAmelCase			(self :					List[str]   ):
    # tokenizer has no padding token
    pass


import logging
import os
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


lowercase__       ={
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING,
    'error': logging.ERROR,
    'critical': logging.CRITICAL,
}

lowercase__       =logging.WARNING




def    __UpperCamelCase     (				):
 __a :  Optional[Any]						=  os.getenv('''DATASETS_VERBOSITY'''  ,   lowerCAmelCase__  )
 if env_level_str:
  if env_level_str in log_levels:
   return log_levels[env_level_str]
  else:
   logging.getLogger().warning(
       f"Unknown option DATASETS_VERBOSITY={env_level_str}, "
       f"has to be one of: { ', '.join(log_levels.keys()  ) }"  )
 return _default_log_level




def    __UpperCamelCase     (				):
 return __name__.split('''.'''  )[0]




def    __UpperCamelCase     (				):
 return logging.getLogger(_get_library_name()  )




def    __UpperCamelCase     (				):
 # Apply our default configuration to the library root logger.
 __a :  str						=  _get_library_root_logger()
 library_root_logger.setLevel(_get_default_logging_level()  )




def    __UpperCamelCase     (				):
 __a :  Any						=  _get_library_root_logger()
 library_root_logger.setLevel(logging.NOTSET  )




def    __UpperCamelCase     (				lowerCAmelCase__							:				Optional[str] = None  ):
 if name is None:
  __a :  Union[str, Any]						=  _get_library_name()
 return logging.getLogger(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return _get_library_root_logger().getEffectiveLevel()




def    __UpperCamelCase     (				lowerCAmelCase__							:				int  ):
 _get_library_root_logger().setLevel(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return set_verbosity(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return set_verbosity(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return set_verbosity(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 return set_verbosity(lowerCAmelCase__  )




def    __UpperCamelCase     (				):
 __a :  Union[str, Any]						=  False




def    __UpperCamelCase     (				):
 __a :  Tuple						=  True


# Configure the library root logger at the module level (singleton-like)
_configure_library_root_logger()



class 		UpperCamelCase__  :
   def __init__(self :					str					,				*snake_case_ :					str					,				**snake_case_ :					Union[str, Any]   ):  # pylint: disable=unused-argument
    __a :  Optional[Any]						=  args[0] if args else None
   def __iter__(self :					List[str]   ):
    return iter(self._iterator   )
   def __getattr__(self :					str					,				snake_case_ :					Optional[Any]   ):

    def empty_fn(*snake_case_ :					int					,				**snake_case_ :					int   ):  # pylint: disable=unused-argument
     return

    return empty_fn
   def __enter__(self :					Union[str, Any]   ):
    return self







   def __exit__(self :					str					,				snake_case_ :					List[str]					,				snake_case_ :					int					,				snake_case_ :					Optional[Any]   ):
    return





lowercase__       =True



class 		UpperCamelCase__  :
   def __call__(self :					Tuple					,				*snake_case_ :					str					,				snake_case_ :					str=False					,				**snake_case_ :					Dict   ):
    if _tqdm_active and not disable:
     return tqdm_lib.tqdm(*snake_case_					,				**snake_case_   )
    else:
     return EmptyTqdm(*snake_case_					,				**snake_case_   )
   def       lowerCAmelCase			(self :					Optional[Any]					,				*snake_case_ :					Union[str, Any]					,				**snake_case_ :					Optional[Any]   ):
    __a :  List[Any]						=  None
    if _tqdm_active:
     return tqdm_lib.tqdm.set_lock(*snake_case_					,				**snake_case_   )







   def       lowerCAmelCase			(self :					str   ):
    if _tqdm_active:
     return tqdm_lib.tqdm.get_lock()





lowercase__       =_tqdm_cls()




def    __UpperCamelCase     (				):
 global _tqdm_active
 return bool(_tqdm_active  )




def    __UpperCamelCase     (				):
 global _tqdm_active
 __a :  Dict						=  True




def    __UpperCamelCase     (				):
 global _tqdm_active
 __a :  Union[str, Any]						=  False
