code: 
  
  
  
  
  
  import json
  import os
  import unittest
  
  from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
      VOCAB_FILES_NAMES,
      GPTNeoXJapaneseTokenizer,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  @require_tokenizers
  class     __a						(					A__ ,     unittest.TestCase ):
   _lowerCAmelCase : str 				=				GPTNeoXJapaneseTokenizer
   _lowerCAmelCase : Dict 				=				False
   _lowerCAmelCase : List[Any] 				=				{'''do_clean_text''': False, '''add_prefix_space''': False}
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     List[Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        super().setUp()
  
        UpperCamelCase__    :     str        =						[
            "こん",
            "こんに",
            "にちは",
            "ばんは",
            "世界,㔺界",
            "、",
            "。",
            "<BR>",
            "<SP>",
            "<TAB>",
            "<URL>",
            "<EMAIL>",
            "<TEL>",
            "<DATE>",
            "<PRICE>",
            "<BLOCK>",
            "<KIGOU>",
            "<U2000U2BFF>",
            "<|emoji1|>",
            "<unk>",
            "<|startoftext|>",
            "<|endoftext|>",
        ]
        UpperCamelCase__    :     int        =						{"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
        UpperCamelCase__    :     Dict        =						{"unk_token": "<unk>"}
  
        UpperCamelCase__    :     Dict        =						os.path.join(self.tmpdirname							,     VOCAB_FILES_NAMES["vocab_file"]		)
        UpperCamelCase__    :     List[Any]        =						os.path.join(self.tmpdirname							,     VOCAB_FILES_NAMES["emoji_file"]		)
        with open(self.vocab_file							,     "w"							,     encoding="utf-8"		) as vocab_writer:
             vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)
        with open(self.emoji_file							,     "w"		) as emoji_writer:
             emoji_writer.write(json.dumps(SCREAMING_SNAKE_CASE		)		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Any							,     **SCREAMING_SNAKE_CASE      :     List[str]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        kwargs.update(self.special_tokens_map		)
        return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname							,     **SCREAMING_SNAKE_CASE		)
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Tuple							,     SCREAMING_SNAKE_CASE      :     List[str]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     List[str]        =						"こんにちは、世界。 \nこんばんは、㔺界。😀"
        UpperCamelCase__    :     Tuple        =						"こんにちは、世界。 \nこんばんは、世界。😀"
        return input_text, output_text
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Any							,     SCREAMING_SNAKE_CASE      :     Tuple		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     str        =						self.get_input_output_texts(SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Any        =						tokenizer.encode(SCREAMING_SNAKE_CASE							,     add_special_tokens=SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     str        =						tokenizer.decode(SCREAMING_SNAKE_CASE							,     clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE		)
        return text, ids
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Optional[Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        pass  # TODO add if relevant
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Dict		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        pass  # TODO add if relevant
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Optional[int]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        pass  # TODO add if relevant
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     List[Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     Dict        =						self.get_tokenizer()
  
        # Testing tokenization
        UpperCamelCase__    :     Union[str, Any]        =						"こんにちは、世界。　こんばんは、㔺界。"
        UpperCamelCase__    :     Optional[Any]        =						["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
        UpperCamelCase__    :     Union[str, Any]        =						tokenizer.tokenize(SCREAMING_SNAKE_CASE		)
        self.assertListEqual(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
  
        # Testing conversion to ids without special tokens
        UpperCamelCase__    :     str        =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
        UpperCamelCase__    :     Optional[int]        =						tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE		)
        self.assertListEqual(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
  
        # Testing conversion to ids with special tokens
        UpperCamelCase__    :     Any        =						tokens + [tokenizer.unk_token]
        UpperCamelCase__    :     Union[str, Any]        =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
        UpperCamelCase__    :     List[str]        =						tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE		)
        self.assertListEqual(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
  
  
  
  
  
  
  
   @slow
   def       __lowercase      (	self      :     Union[str, Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        UpperCamelCase__    :     List[Any]        =						self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b"		)
  
        UpperCamelCase__    :     Dict        =						tokenizer.encode("ありがとう。"							,     add_special_tokens=SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     List[str]        =						tokenizer.encode("どういたしまして。"							,     add_special_tokens=SCREAMING_SNAKE_CASE		)
  
        UpperCamelCase__    :     int        =						tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE		)
        UpperCamelCase__    :     Optional[int]        =						tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE							,     SCREAMING_SNAKE_CASE		)
  
        assert encoded_sentence == ids_a
        assert encoded_pair == ids_a + ids_a
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Optional[Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        pass
  
  
  
  
  
  
  
   def       __lowercase      (	self      :     Optional[Any]		):
  
  
        '''simple docstring'''
  
  
  
  
  
  
        pass
style_context: 
  
  
  
  
  
  
  from __future__ import annotations
  
  
  
  
  
  
  
  def 	SCREAMING_SNAKE_CASE							(      __lowerCAmelCase    )				->			list[int]:
       assert (
           isinstance(__lowerCAmelCase      ,					__lowerCAmelCase    ) and mask > 0
       ), f'mask needs to be positive integer, your input {mask}'
       UpperCamelCase__    :     Any        =						[]
       UpperCamelCase__    :     int        =						mask
  
       while submask:
            all_submasks.append(__lowerCAmelCase    )
            UpperCamelCase__    :     List[str]        =						(submask - 1) & mask
  
       return all_submasks
  
  
  if __name__ == "__main__":
   import doctest
  
   doctest.testmod()