code: 
  
  
  
  '''simple docstring'''
  
  
  
  import json
  import os
  import unittest
  
  from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
      VOCAB_FILES_NAMES,
      GPTNeoXJapaneseTokenizer,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin
  @require_tokenizers
  class        UpperCAmelCase__  (    UpperCAmelCase_							,			unittest.TestCase):
   __SCREAMING_SNAKE_CASE					      =				GPTNeoXJapaneseTokenizer
   __SCREAMING_SNAKE_CASE					      =				False
   __SCREAMING_SNAKE_CASE					      =				{'''do_clean_text''': False, '''add_prefix_space''': False}
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Optional[Any]:
    super().setUp()
  
    __UpperCamelCase				      =  [
        """こん""",
        """こんに""",
        """にちは""",
        """ばんは""",
        """世界,㔺界""",
        """、""",
        """。""",
        """<BR>""",
        """<SP>""",
        """<TAB>""",
        """<URL>""",
        """<EMAIL>""",
        """<TEL>""",
        """<DATE>""",
        """<PRICE>""",
        """<BLOCK>""",
        """<KIGOU>""",
        """<U2000U2BFF>""",
        """<|emoji1|>""",
        """<unk>""",
        """<|startoftext|>""",
        """<|endoftext|>""",
    ]
    __UpperCamelCase				      =  {"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
    __UpperCamelCase				      =  {"""unk_token""": """<unk>"""}
  
    __UpperCamelCase				      =  os.path.join(self.tmpdirname		,	VOCAB_FILES_NAMES["""vocab_file"""]			)
    __UpperCamelCase				      =  os.path.join(self.tmpdirname		,	VOCAB_FILES_NAMES["""emoji_file"""]			)
    with open(self.vocab_file		,	"""w"""		,	encoding="""utf-8"""			) as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]			)			)
    with open(self.emoji_file		,	"""w"""			) as emoji_writer:
     emoji_writer.write(json.dumps(lowercase			)			)
  
  
  
   def    __lowerCamelCase				(  self		,	**lowercase			)  ->							Tuple:
    kwargs.update(self.special_tokens_map			)
    return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname		,	**lowercase			)
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase			)  ->							Dict:
    __UpperCamelCase				      =  """こんにちは、世界。 \nこんばんは、㔺界。😀"""
    __UpperCamelCase				      =  """こんにちは、世界。 \nこんばんは、世界。😀"""
    return input_text, output_text
  
  
  
   def    __lowerCamelCase				(  self		,	lowercase			)  ->							List[Any]:
    __UpperCamelCase      ,		__UpperCamelCase				      =  self.get_input_output_texts(lowercase			)
    __UpperCamelCase				      =  tokenizer.encode(lowercase		,	add_special_tokens=lowercase			)
    __UpperCamelCase				      =  tokenizer.decode(lowercase		,	clean_up_tokenization_spaces=lowercase			)
    return text, ids
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Optional[int]:
    pass  # TODO add if relevant
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Tuple:
    pass  # TODO add if relevant
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Optional[Any]:
    pass  # TODO add if relevant
  
  
  
   def    __lowerCamelCase				(  self			)  ->							int:
    __UpperCamelCase				      =  self.get_tokenizer()
  
    # Testing tokenization
    __UpperCamelCase				      =  """こんにちは、世界。　こんばんは、㔺界。"""
    __UpperCamelCase				      =  ["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
    __UpperCamelCase				      =  tokenizer.tokenize(lowercase			)
    self.assertListEqual(lowercase		,	lowercase			)
  
    # Testing conversion to ids without special tokens
    __UpperCamelCase				      =  [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
    __UpperCamelCase				      =  tokenizer.convert_tokens_to_ids(lowercase			)
    self.assertListEqual(lowercase		,	lowercase			)
  
    # Testing conversion to ids with special tokens
    __UpperCamelCase				      =  tokens + [tokenizer.unk_token]
    __UpperCamelCase				      =  [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
    __UpperCamelCase				      =  tokenizer.convert_tokens_to_ids(lowercase			)
    self.assertListEqual(lowercase		,	lowercase			)
  
  
  
   @slow
   def    __lowerCamelCase				(  self			)  ->							Optional[int]:
    __UpperCamelCase				      =  self.tokenizer_class.from_pretrained("""abeja/gpt-neox-japanese-2.7b"""			)
  
    __UpperCamelCase				      =  tokenizer.encode("""ありがとう。"""		,	add_special_tokens=lowercase			)
    __UpperCamelCase				      =  tokenizer.encode("""どういたしまして。"""		,	add_special_tokens=lowercase			)
  
    __UpperCamelCase				      =  tokenizer.build_inputs_with_special_tokens(lowercase			)
    __UpperCamelCase				      =  tokenizer.build_inputs_with_special_tokens(lowercase		,	lowercase			)
  
    assert encoded_sentence == ids_a
    assert encoded_pair == ids_a + ids_a
  
  
  
   def    __lowerCamelCase				(  self			)  ->							Dict:
    # Intentionally convert some words to accommodate character fluctuations unique to Japanese
    pass
  
  
  
  
   def    __lowerCamelCase				(  self			)  ->							List[str]:
    # tokenizer has no padding token
    pass
  
  
  
  
  
  
style_context: 
  
  
  
  '''simple docstring'''
  
  
  
  def _lowercase   (  __A							,__A       ):
  
  
  
   '''simple docstring'''
  
  
  
   __UpperCamelCase				      =  1
   while b > 0:
    if b & 1:
     res *= a
  
    a *= a
    b >>= 1
  
   return res
  
  
  
  
  
  
  
  def _lowercase   (  __A							,__A							,__A       ):
  
  
  
   '''simple docstring'''
  
  
  
   __UpperCamelCase				      =  1
   while b > 0:
    if b & 1:
     __UpperCamelCase				      =  ((res % c) * (a % c)) % c
  
    a *= a
    b >>= 1
  
   return res
  
  
  
  
  
  