





'''simple docstring'''


import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin
@custom_tokenizers
class       a_   (				SCREAMING_SNAKE_CASE__	,						unittest.TestCase							):
     __A      =			BertJapaneseTokenizer
     __A      =			False
     __A      =			True





     def 							lowercase__      ( self		:					int						):





        """simple docstring"""




        super().setUp()

        lowercase_			:int    			= [
            '''[UNK]''',
            '''[CLS]''',
            '''[SEP]''',
            '''こんにちは''',
            '''こん''',
            '''にちは''',
            '''ばんは''',
            '''##こん''',
            '''##にちは''',
            '''##ばんは''',
            '''世界''',
            '''##世界''',
            '''、''',
            '''##、''',
            '''。''',
            '''##。''',
        ]

        lowercase_			:List[str]    			= os.path.join(self.tmpdirname	,      VOCAB_FILES_NAMES["vocab_file"]						)
        with open(self.vocab_file	,      "w"	,      encoding="utf-8"						) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)





     def 							lowercase__      ( self		:					str	,      lowercase		:					Any						):





        """simple docstring"""




        lowercase_			:Optional[int]    			= '''こんにちは、世界。 \nこんばんは、世界。'''
        lowercase_			:Optional[int]    			= '''こんにちは 、 世界 。 こんばんは 、 世界 。'''
        return input_text, output_text





     def 							lowercase__      ( self		:					Union[str, Any]	,      lowercase		:					List[Any]						):





        """simple docstring"""




        lowercase_			:Tuple    			= self.get_input_output_texts(__lowerCamelCase						)
        lowercase_			:Optional[Any]    			= tokenizer.encode(__lowerCamelCase	,      add_special_tokens=__lowerCamelCase						)
        lowercase_			:Tuple    			= tokenizer.decode(__lowerCamelCase	,      clean_up_tokenization_spaces=__lowerCamelCase						)
        return text, ids





     def 							lowercase__      ( self		:					str						):





        """simple docstring"""




        pass  # TODO add if relevant





     def 							lowercase__      ( self		:					Optional[int]						):





        """simple docstring"""




        pass  # TODO add if relevant





     def 							lowercase__      ( self		:					List[Any]						):





        """simple docstring"""




        pass  # TODO add if relevant





     def 							lowercase__      ( self		:					Optional[int]						):





        """simple docstring"""




        lowercase_			:str    			= self.tokenizer_class(self.vocab_file						)

        lowercase_			:List[Any]    			= tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"						)
        self.assertListEqual(__lowerCamelCase	,      ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]						)
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase						)	,      [3, 12, 10, 14, 4, 9, 12, 10, 14]						)





     def 							lowercase__      ( self		:					List[str]						):





        """simple docstring"""




        lowercase_			:Tuple    			= self.tokenizer_class(self.vocab_file	,      word_tokenizer_type="mecab"						)
        self.assertIsNotNone(__lowerCamelCase						)

        lowercase_			:Optional[int]    			= '''こんにちは、世界。\nこんばんは、世界。'''
        lowercase_			:Tuple    			= tokenizer.tokenize(__lowerCamelCase						)
        self.assertListEqual(__lowerCamelCase	,      ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]						)
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase						)	,      [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

        lowercase_			:Any    			= os.path.join(self.tmpdirname	,      "tokenizer.bin"						)
        with open(__lowerCamelCase	,      "wb"						) as handle:
           pickle.dump(__lowerCamelCase	,      __lowerCamelCase						)

        with open(__lowerCamelCase	,      "rb"						) as handle:
           lowercase_			:Union[str, Any]    			= pickle.load(__lowerCamelCase						)

        lowercase_			:str    			= tokenizer_new.tokenize(__lowerCamelCase						)

        self.assertListEqual(__lowerCamelCase	,      __lowerCamelCase						)





     def 							lowercase__      ( self		:					Tuple						):





        """simple docstring"""




        lowercase_			:Any    			= MecabTokenizer(mecab_dic="ipadic"						)

        self.assertListEqual(
            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)	,      ["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]	,      )





     def 							lowercase__      ( self		:					Any						):





        """simple docstring"""




        try:
           lowercase_			:int    			= MecabTokenizer(mecab_dic="unidic_lite"						)
        except ModuleNotFoundError:
           return

        self.assertListEqual(
            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)	,      ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]	,      )





     def 							lowercase__      ( self		:					List[Any]						):





        """simple docstring"""




        try:
           lowercase_			:List[Any]    			= MecabTokenizer(mecab_dic="unidic"						)
        except ModuleNotFoundError:
           return

        self.assertListEqual(
            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)	,      ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]	,      )





     def 							lowercase__      ( self		:					str						):





        """simple docstring"""




        lowercase_			:Dict    			= MecabTokenizer(do_lower_case=__lowerCamelCase	,      mecab_dic="ipadic"						)

        self.assertListEqual(
            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)	,      ["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]	,      )





     def 							lowercase__      ( self		:					Union[str, Any]						):





        """simple docstring"""




        try:
           lowercase_			:str    			= MecabTokenizer(
               do_lower_case=__lowerCamelCase	,      normalize_text=__lowerCamelCase	,      mecab_option="-d /usr/local/lib/mecab/dic/jumandic"						)
        except RuntimeError:
           # if dict doesn't exist in the system, previous code raises this error.
           return

        self.assertListEqual(
            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)	,      ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]	,      )





     def 							lowercase__      ( self		:					str						):





        """simple docstring"""




        lowercase_			:List[Any]    			= MecabTokenizer(normalize_text=__lowerCamelCase	,      mecab_dic="ipadic"						)

        self.assertListEqual(
            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)	,      ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]	,      )





     @require_sudachi
     def 							lowercase__      ( self		:					Optional[int]						):





        """simple docstring"""




        lowercase_			:Optional[int]    			= self.tokenizer_class(self.vocab_file	,      word_tokenizer_type="sudachi"						)
        self.assertIsNotNone(__lowerCamelCase						)

        lowercase_			:Any    			= '''こんにちは、世界。\nこんばんは、世界。'''
        lowercase_			:Any    			= tokenizer.tokenize(__lowerCamelCase						)
        self.assertListEqual(__lowerCamelCase	,      ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]						)
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase						)	,      [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

        lowercase_			:str    			= os.path.join(self.tmpdirname	,      "tokenizer.bin"						)
        with open(__lowerCamelCase	,      "wb"						) as handle:
           pickle.dump(__lowerCamelCase	,      __lowerCamelCase						)

        with open(__lowerCamelCase	,      "rb"						) as handle:
           lowercase_			:Optional[Any]    			= pickle.load(__lowerCamelCase						)

        lowercase_			:int    			= tokenizer_new.tokenize(__lowerCamelCase						)

        self.assertListEqual(__lowerCamelCase	,      __lowerCamelCase						)





     @require_sudachi
     def 							lowercase__      ( self		:					int						):





        """simple docstring"""




        lowercase_			:Optional[int]    			= SudachiTokenizer(sudachi_dict_type="core"						)

        self.assertListEqual(
            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)	,      [" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]	,      )





     @require_sudachi
     def 							lowercase__      ( self		:					List[Any]						):





        """simple docstring"""




        lowercase_			:Dict    			= SudachiTokenizer(sudachi_dict_type="core"	,      sudachi_split_mode="A"						)

        self.assertListEqual(tokenizer.tokenize("外国人参政権"						)	,      ["外国", "人", "参政", "権"]						)





     @require_sudachi
     def 							lowercase__      ( self		:					Optional[Any]						):





        """simple docstring"""




        lowercase_			:Any    			= SudachiTokenizer(sudachi_dict_type="core"	,      sudachi_split_mode="B"						)

        self.assertListEqual(tokenizer.tokenize("外国人参政権"						)	,      ["外国人", "参政権"]						)





     @require_sudachi
     def 							lowercase__      ( self		:					str						):





        """simple docstring"""




        lowercase_			:Any    			= SudachiTokenizer(sudachi_dict_type="core"	,      sudachi_split_mode="C"						)

        self.assertListEqual(tokenizer.tokenize("外国人参政権"						)	,      ["外国人参政権"]						)





     @require_sudachi
     def 							lowercase__      ( self		:					str						):





        """simple docstring"""




        lowercase_			:List[str]    			= SudachiTokenizer(do_lower_case=__lowerCamelCase	,      sudachi_dict_type="core"						)

        self.assertListEqual(
            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)	,      [" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]	,      )





     @require_sudachi
     def 							lowercase__      ( self		:					List[str]						):





        """simple docstring"""




        lowercase_			:List[Any]    			= SudachiTokenizer(normalize_text=__lowerCamelCase	,      sudachi_dict_type="core"						)

        self.assertListEqual(
            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)	,      [" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]	,      )





     @require_sudachi
     def 							lowercase__      ( self		:					List[Any]						):





        """simple docstring"""




        lowercase_			:str    			= SudachiTokenizer(trim_whitespace=__lowerCamelCase	,      sudachi_dict_type="core"						)

        self.assertListEqual(
            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)	,      ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]	,      )





     @require_jumanpp
     def 							lowercase__      ( self		:					List[str]						):





        """simple docstring"""




        lowercase_			:str    			= self.tokenizer_class(self.vocab_file	,      word_tokenizer_type="jumanpp"						)
        self.assertIsNotNone(__lowerCamelCase						)

        lowercase_			:Union[str, Any]    			= '''こんにちは、世界。\nこんばんは、世界。'''
        lowercase_			:Dict    			= tokenizer.tokenize(__lowerCamelCase						)
        self.assertListEqual(__lowerCamelCase	,      ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]						)
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase						)	,      [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

        lowercase_			:Tuple    			= os.path.join(self.tmpdirname	,      "tokenizer.bin"						)
        with open(__lowerCamelCase	,      "wb"						) as handle:
           pickle.dump(__lowerCamelCase	,      __lowerCamelCase						)

        with open(__lowerCamelCase	,      "rb"						) as handle:
           lowercase_			:List[str]    			= pickle.load(__lowerCamelCase						)

        lowercase_			:List[Any]    			= tokenizer_new.tokenize(__lowerCamelCase						)

        self.assertListEqual(__lowerCamelCase	,      __lowerCamelCase						)





     @require_jumanpp
     def 							lowercase__      ( self		:					str						):





        """simple docstring"""




        lowercase_			:Union[str, Any]    			= JumanppTokenizer()

        self.assertListEqual(
            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)	,      ["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]	,      )





     @require_jumanpp
     def 							lowercase__      ( self		:					Optional[Any]						):





        """simple docstring"""




        lowercase_			:Any    			= JumanppTokenizer(do_lower_case=__lowerCamelCase						)

        self.assertListEqual(
            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)	,      ["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]	,      )





     @require_jumanpp
     def 							lowercase__      ( self		:					Tuple						):





        """simple docstring"""




        lowercase_			:Union[str, Any]    			= JumanppTokenizer(normalize_text=__lowerCamelCase						)

        self.assertListEqual(
            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)	,      ["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]	,      )





     @require_jumanpp
     def 							lowercase__      ( self		:					List[Any]						):





        """simple docstring"""




        lowercase_			:int    			= JumanppTokenizer(trim_whitespace=__lowerCamelCase						)

        self.assertListEqual(
            tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "						)	,      ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]	,      )





     @require_jumanpp
     def 							lowercase__      ( self		:					Optional[int]						):





        """simple docstring"""




        lowercase_			:Optional[Any]    			= JumanppTokenizer()

        self.assertListEqual(
            tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"						)	,      ["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]	,      )





     def 							lowercase__      ( self		:					Union[str, Any]						):





        """simple docstring"""




        lowercase_			:int    			= ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こんにちは''', '''こん''', '''にちは''', '''ばんは''', '''##こん''', '''##にちは''', '''##ばんは''']

        lowercase_			:Optional[int]    			= {}
        for i, token in enumerate(__lowerCamelCase						):
           lowercase_			:Optional[int]    			= i
        lowercase_			:List[str]    			= WordpieceTokenizer(vocab=__lowerCamelCase	,      unk_token="[UNK]"						)

        self.assertListEqual(tokenizer.tokenize(""						)	,      []						)

        self.assertListEqual(tokenizer.tokenize("こんにちは"						)	,      ["こんにちは"]						)

        self.assertListEqual(tokenizer.tokenize("こんばんは"						)	,      ["こん", "##ばんは"]						)

        self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"						)	,      ["こん", "##ばんは", "[UNK]", "こんにちは"]						)





     def 							lowercase__      ( self		:					Dict						):





        """simple docstring"""




        lowercase_			:Union[str, Any]    			= BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"						)
        lowercase_			:List[str]    			= tokenizer.subword_tokenizer

        lowercase_			:List[str]    			= subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"						)
        self.assertListEqual(__lowerCamelCase	,      ["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]						)

        lowercase_			:Tuple    			= subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"						)
        self.assertListEqual(__lowerCamelCase	,      ["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]						)







     def 							lowercase__      ( self		:					Tuple						):





        """simple docstring"""




        lowercase_			:Optional[int]    			= self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"						)

        lowercase_			:Union[str, Any]    			= tokenizer.encode("ありがとう。"	,      add_special_tokens=__lowerCamelCase						)
        lowercase_			:Tuple    			= tokenizer.encode("どういたしまして。"	,      add_special_tokens=__lowerCamelCase						)

        lowercase_			:Any    			= tokenizer.build_inputs_with_special_tokens(__lowerCamelCase						)
        lowercase_			:List[Any]    			= tokenizer.build_inputs_with_special_tokens(__lowerCamelCase	,      __lowerCamelCase						)

        # 2 is for "[CLS]", 3 is for "[SEP]"
        assert encoded_sentence == [2] + text + [3]
        assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class       a_   (				SCREAMING_SNAKE_CASE__	,						unittest.TestCase							):
     __A      =			BertJapaneseTokenizer
     __A      =			False





     def 							lowercase__      ( self		:					str						):





        """simple docstring"""




        super().setUp()

        lowercase_			:Optional[int]    			= ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

        lowercase_			:Optional[int]    			= os.path.join(self.tmpdirname	,      VOCAB_FILES_NAMES["vocab_file"]						)
        with open(self.vocab_file	,      "w"	,      encoding="utf-8"						) as vocab_writer:
           vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)





     def 							lowercase__      ( self		:					Any	,      **lowercase		:					str						):





        """simple docstring"""




        return BertJapaneseTokenizer.from_pretrained(self.tmpdirname	,      subword_tokenizer_type="character"	,      **__lowerCamelCase						)





     def 							lowercase__      ( self		:					Union[str, Any]	,      lowercase		:					Optional[int]						):





        """simple docstring"""




        lowercase_			:List[Any]    			= '''こんにちは、世界。 \nこんばんは、世界。'''
        lowercase_			:Optional[Any]    			= '''こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'''
        return input_text, output_text





     def 							lowercase__      ( self		:					int						):





        """simple docstring"""




        pass  # TODO add if relevant





     def 							lowercase__      ( self		:					str						):





        """simple docstring"""




        pass  # TODO add if relevant





     def 							lowercase__      ( self		:					Union[str, Any]						):





        """simple docstring"""




        pass  # TODO add if relevant





     def 							lowercase__      ( self		:					str						):





        """simple docstring"""




        lowercase_			:Union[str, Any]    			= self.tokenizer_class(self.vocab_file	,      subword_tokenizer_type="character"						)

        lowercase_			:Optional[int]    			= tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"						)
        self.assertListEqual(
            __lowerCamelCase	,      ["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]						)
        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(__lowerCamelCase						)	,      [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)





     def 							lowercase__      ( self		:					Union[str, Any]						):





        """simple docstring"""




        lowercase_			:str    			= ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''こ''', '''ん''', '''に''', '''ち''', '''は''', '''ば''', '''世''', '''界''', '''、''', '''。''']

        lowercase_			:str    			= {}
        for i, token in enumerate(__lowerCamelCase						):
           lowercase_			:Union[str, Any]    			= i
        lowercase_			:Optional[int]    			= CharacterTokenizer(vocab=__lowerCamelCase	,      unk_token="[UNK]"						)

        self.assertListEqual(tokenizer.tokenize(""						)	,      []						)

        self.assertListEqual(tokenizer.tokenize("こんにちは"						)	,      ["こ", "ん", "に", "ち", "は"]						)

        self.assertListEqual(tokenizer.tokenize("こんにちほ"						)	,      ["こ", "ん", "に", "ち", "[UNK]"]						)







     def 							lowercase__      ( self		:					Optional[Any]						):





        """simple docstring"""




        lowercase_			:Union[str, Any]    			= self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"						)

        lowercase_			:List[Any]    			= tokenizer.encode("ありがとう。"	,      add_special_tokens=__lowerCamelCase						)
        lowercase_			:List[Any]    			= tokenizer.encode("どういたしまして。"	,      add_special_tokens=__lowerCamelCase						)

        lowercase_			:Optional[Any]    			= tokenizer.build_inputs_with_special_tokens(__lowerCamelCase						)
        lowercase_			:Union[str, Any]    			= tokenizer.build_inputs_with_special_tokens(__lowerCamelCase	,      __lowerCamelCase						)

        # 2 is for "[CLS]", 3 is for "[SEP]"
        assert encoded_sentence == [2] + text + [3]
        assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class       a_   (				unittest.TestCase							):





     def 							lowercase__      ( self		:					Tuple						):





        """simple docstring"""




        lowercase_			:Optional[Any]    			= '''cl-tohoku/bert-base-japanese'''
        lowercase_			:List[Any]    			= AutoTokenizer.from_pretrained(__lowerCamelCase						)
        self.assertIsInstance(__lowerCamelCase	,      __lowerCamelCase						)







class       a_   (				unittest.TestCase							):





     def 							lowercase__      ( self		:					str						):





        """simple docstring"""




        lowercase_			:List[str]    			= '''cl-tohoku/bert-base-japanese'''
        with self.assertLogs("transformers"	,      level="WARNING"						) as cm:
           BertTokenizer.from_pretrained(__lowerCamelCase						)
           self.assertTrue(
               cm.records[0].message.startswith(
                   "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                   " is called from."						)						)
        lowercase_			:List[str]    			= '''bert-base-cased'''
        with self.assertLogs("transformers"	,      level="WARNING"						) as cm:
           BertJapaneseTokenizer.from_pretrained(__lowerCamelCase						)
           self.assertTrue(
               cm.records[0].message.startswith(
                   "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                   " is called from."						)						)









"""simple docstring"""



import random
import unittest

import numpy as np

import transformers
from transformers import is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax


if is_flax_available():
       import os

       import jax.numpy as jnp
       from jax import jit

       from transformers import AutoTokenizer, FlaxAutoModelForCausalLM
       from transformers.modeling_flax_pytorch_utils import load_flax_weights_in_pytorch_model

       a_ =			"""0.12"""  # assumed parallelism: 8


if is_torch_available():
       import torch







def 		__lowercase     (   snake_case_  :  Any				,snake_case_  :  int				,snake_case_  :  List[str]=None						)   ->Union[str, Any]:



    '''simple docstring'''







    if rng is None:
        __A    :						Union[str, Any]						 =     random.Random()

    __A    :						List[str]						 =     1
    for dim in shape:
        total_dims *= dim

    __A    :						List[str]						 =     []
    for _ in range(snake_case_						):
        values.append(rng.randint(0				,vocab_size - 1						)						)

    __A    :						Optional[Any]						 =     np.array(snake_case_				,dtype=jnp.intaa						).reshape(snake_case_						)

    return output







def 		__lowercase     (   snake_case_  :  int				,snake_case_  :  Union[str, Any]=None						)   ->int:



    '''simple docstring'''







    __A    :						int						 =     ids_tensor(snake_case_				,vocab_size=2				,rng=snake_case_						)
    # make sure that at least one token is attended to for each batch
    __A    :						str						 =     1
    return attn_mask






@require_flax
class 		__snake_case					:





 """simple docstring"""


 _lowerCamelCase        =     None
 _lowerCamelCase        =     ()

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A ,       __A    :						Optional[int]						 =     self.model_tester.prepare_config_and_inputs_for_common()

     # cut to half length & take max batch_size 3
     __A    :						Optional[int]						 =     2
     __A    :						Tuple						 =     inputs['''input_ids'''].shape[-1] // 2
     __A    :						List[Any]						 =     inputs['''input_ids'''][:max_batch_size, :sequence_length]

     __A    :						Optional[Any]						 =     jnp.ones_like(__lowerCamelCase  )
     __A    :						List[Any]						 =     attention_mask[:max_batch_size, :sequence_length]

     # generate max 5 tokens
     __A    :						Optional[int]						 =     input_ids.shape[-1] + 5
     if config.eos_token_id is not None and config.pad_token_id is None:
         # hack to allow generate for models such as GPT2 as is done in `generate()`
         __A    :						Optional[int]						 =     config.eos_token_id
     return config, input_ids, attention_mask, max_length

 @is_pt_flax_cross_test
 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A ,       __A ,       __A ,       __A    :						List[Any]						 =     self._get_input_ids_and_config()
     __A    :						List[Any]						 =     False
     __A    :						str						 =     max_length
     __A    :						Tuple						 =     0

     for model_class in self.all_generative_model_classes:
         __A    :						Union[str, Any]						 =     model_class(__lowerCamelCase  )

         __A    :						Any						 =     model_class.__name__[4:]  # Skip the "Flax" at the beginning
         __A    :						Dict						 =     getattr(__lowerCamelCase					,						__lowerCamelCase  )
         __A    :						Tuple						 =     pt_model_class(__lowerCamelCase  ).eval()
         __A    :						Optional[Any]						 =     load_flax_weights_in_pytorch_model(__lowerCamelCase					,						flax_model.params  )

         __A    :						str						 =     flax_model.generate(__lowerCamelCase  ).sequences
         __A    :						List[Any]						 =     pt_model.generate(torch.tensor(__lowerCamelCase					,						dtype=torch.long  )  )

         if flax_generation_outputs.shape[-1] > pt_generation_outputs.shape[-1]:
             __A    :						Any						 =     flax_generation_outputs[:, : pt_generation_outputs.shape[-1]]

         self.assertListEqual(pt_generation_outputs.numpy().tolist()					,						flax_generation_outputs.tolist()  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A ,       __A ,       __A ,       __A    :						Union[str, Any]						 =     self._get_input_ids_and_config()
     __A    :						List[str]						 =     False
     __A    :						Optional[Any]						 =     max_length

     for model_class in self.all_generative_model_classes:
         __A    :						Optional[int]						 =     model_class(__lowerCamelCase  )

         __A    :						Dict						 =     model.generate(__lowerCamelCase  ).sequences
         self.assertEqual(generation_outputs.shape[-1]					,						__lowerCamelCase  )

         __A    :						Union[str, Any]						 =     jit(model.generate  )
         __A    :						str						 =     jit_generate(__lowerCamelCase  ).sequences

         self.assertListEqual(generation_outputs.tolist()					,						jit_generation_outputs.tolist()  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A ,       __A ,       __A ,       __A    :						int						 =     self._get_input_ids_and_config()
     __A    :						List[Any]						 =     True
     __A    :						Union[str, Any]						 =     max_length

     for model_class in self.all_generative_model_classes:
         __A    :						Dict						 =     model_class(__lowerCamelCase  )

         __A    :						List[Any]						 =     model.generate(__lowerCamelCase  ).sequences
         self.assertEqual(generation_outputs.shape[-1]					,						__lowerCamelCase  )

         __A    :						List[Any]						 =     jit(model.generate  )
         __A    :						List[str]						 =     jit_generate(__lowerCamelCase  ).sequences

         self.assertListEqual(generation_outputs.tolist()					,						jit_generation_outputs.tolist()  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A ,       __A ,       __A ,       __A    :						List[Any]						 =     self._get_input_ids_and_config()
     __A    :						List[str]						 =     False
     __A    :						Any						 =     max_length
     __A    :						List[Any]						 =     2

     for model_class in self.all_generative_model_classes:
         __A    :						Tuple						 =     model_class(__lowerCamelCase  )

         __A    :						Optional[Any]						 =     model.generate(__lowerCamelCase  ).sequences
         self.assertEqual(generation_outputs.shape[-1]					,						__lowerCamelCase  )

         __A    :						Union[str, Any]						 =     jit(model.generate  )
         __A    :						Optional[int]						 =     jit_generate(__lowerCamelCase  ).sequences

         self.assertListEqual(generation_outputs.tolist()					,						jit_generation_outputs.tolist()  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A ,       __A ,       __A ,       __A    :						Tuple						 =     self._get_input_ids_and_config()
     __A    :						str						 =     False
     __A    :						Union[str, Any]						 =     max_length
     __A    :						List[Any]						 =     2
     __A    :						Any						 =     2

     for model_class in self.all_generative_model_classes:
         __A    :						List[str]						 =     model_class(__lowerCamelCase  )

         __A    :						Optional[int]						 =     model.generate(__lowerCamelCase  ).sequences
         self.assertEqual(generation_outputs.shape[0]					,						input_ids.shape[0] * config.num_return_sequences  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A ,       __A ,       __A ,       __A    :						Any						 =     self._get_input_ids_and_config()
     __A    :						Optional[Any]						 =     True
     __A    :						Union[str, Any]						 =     max_length
     __A    :						List[str]						 =     0.8
     __A    :						List[str]						 =     10
     __A    :						Union[str, Any]						 =     0.3
     __A    :						Union[str, Any]						 =     1
     __A    :						Optional[Any]						 =     8
     __A    :						Dict						 =     9

     for model_class in self.all_generative_model_classes:
         __A    :						List[Any]						 =     model_class(__lowerCamelCase  )

         __A    :						List[str]						 =     model.generate(__lowerCamelCase  ).sequences
         self.assertEqual(generation_outputs.shape[-1]					,						__lowerCamelCase  )

         __A    :						Any						 =     jit(model.generate  )
         __A    :						str						 =     jit_generate(__lowerCamelCase  ).sequences

         self.assertListEqual(generation_outputs.tolist()					,						jit_generation_outputs.tolist()  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A ,       __A ,       __A ,       __A    :						Union[str, Any]						 =     self._get_input_ids_and_config()
     __A    :						Union[str, Any]						 =     max_length
     __A    :						List[str]						 =     1
     __A    :						str						 =     8
     __A    :						Any						 =     9

     for model_class in self.all_generative_model_classes:
         __A    :						Union[str, Any]						 =     model_class(__lowerCamelCase  )

         __A    :						Tuple						 =     model.generate(__lowerCamelCase  ).sequences
         self.assertEqual(generation_outputs.shape[-1]					,						__lowerCamelCase  )

         __A    :						Optional[Any]						 =     jit(model.generate  )
         __A    :						Optional[Any]						 =     jit_generate(__lowerCamelCase  ).sequences

         self.assertListEqual(generation_outputs.tolist()					,						jit_generation_outputs.tolist()  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A ,       __A ,       __A ,       __A    :						int						 =     self._get_input_ids_and_config()
     __A    :						Optional[int]						 =     max_length
     __A    :						List[str]						 =     2
     __A    :						List[Any]						 =     1
     __A    :						Optional[Any]						 =     8
     __A    :						str						 =     9

     for model_class in self.all_generative_model_classes:
         __A    :						Optional[int]						 =     model_class(__lowerCamelCase  )

         __A    :						int						 =     model.generate(__lowerCamelCase  ).sequences
         self.assertEqual(generation_outputs.shape[-1]					,						__lowerCamelCase  )

         __A    :						str						 =     jit(model.generate  )
         __A    :						Any						 =     jit_generate(__lowerCamelCase  ).sequences

         self.assertListEqual(generation_outputs.tolist()					,						jit_generation_outputs.tolist()  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A ,       __A ,       __A ,       __A    :						int						 =     self._get_input_ids_and_config()

     # pad attention mask on the left
     __A    :						Dict						 =     attention_mask.at[(0, 0)].set(0  )

     __A    :						Optional[Any]						 =     False
     __A    :						Union[str, Any]						 =     max_length

     for model_class in self.all_generative_model_classes:
         __A    :						int						 =     model_class(__lowerCamelCase  )

         __A    :						Union[str, Any]						 =     model.generate(__lowerCamelCase					,						attention_mask=__lowerCamelCase  ).sequences
         self.assertEqual(generation_outputs.shape[-1]					,						__lowerCamelCase  )

         __A    :						Optional[int]						 =     jit(model.generate  )
         __A    :						Any						 =     jit_generate(__lowerCamelCase					,						attention_mask=__lowerCamelCase  ).sequences

         self.assertListEqual(generation_outputs.tolist()					,						jit_generation_outputs.tolist()  )

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A ,       __A ,       __A ,       __A    :						Optional[int]						 =     self._get_input_ids_and_config()

     # pad attention mask on the left
     __A    :						str						 =     attention_mask.at[(0, 0)].set(0  )

     __A    :						List[Any]						 =     True
     __A    :						Any						 =     max_length

     for model_class in self.all_generative_model_classes:
         __A    :						str						 =     model_class(__lowerCamelCase  )

         __A    :						List[Any]						 =     model.generate(__lowerCamelCase					,						attention_mask=__lowerCamelCase  ).sequences
         self.assertEqual(generation_outputs.shape[-1]					,						__lowerCamelCase  )

         __A    :						List[str]						 =     jit(model.generate  )
         __A    :						Optional[int]						 =     jit_generate(__lowerCamelCase					,						attention_mask=__lowerCamelCase  ).sequences

         self.assertListEqual(generation_outputs.tolist()					,						jit_generation_outputs.tolist()  )







 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A ,       __A ,       __A ,       __A    :						Optional[Any]						 =     self._get_input_ids_and_config()

     # pad attention mask on the left
     __A    :						List[str]						 =     attention_mask.at[(0, 0)].set(0  )

     __A    :						Optional[int]						 =     2
     __A    :						Dict						 =     max_length

     for model_class in self.all_generative_model_classes:
         __A    :						Any						 =     model_class(__lowerCamelCase  )

         __A    :						int						 =     model.generate(__lowerCamelCase					,						attention_mask=__lowerCamelCase  ).sequences
         self.assertEqual(generation_outputs.shape[-1]					,						__lowerCamelCase  )

         __A    :						str						 =     jit(model.generate  )
         __A    :						str						 =     jit_generate(__lowerCamelCase					,						attention_mask=__lowerCamelCase  ).sequences

         self.assertListEqual(generation_outputs.tolist()					,						jit_generation_outputs.tolist()  )






@require_flax
class 		__snake_case					(		unittest.TestCase     ):





 """simple docstring"""


 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     __A    :						Optional[int]						 =     AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-bert'''  )
     __A    :						Optional[Any]						 =     FlaxAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''  )

     __A    :						Any						 =     '''Hello world'''
     __A    :						Dict						 =     tokenizer(__lowerCamelCase					,						return_tensors='''np'''  ).input_ids

     # typos are quickly detected (the correct argument is `do_sample`)
     with self.assertRaisesRegex(__lowerCamelCase					,						'''do_samples'''  ):
         model.generate(__lowerCamelCase					,						do_samples=__lowerCamelCase  )

     # arbitrary arguments that will not be used anywhere are also not accepted
     with self.assertRaisesRegex(__lowerCamelCase					,						'''foo'''  ):
         __A    :						Any						 =     {'''foo''': '''bar'''}
         model.generate(__lowerCamelCase					,						**__lowerCamelCase  )




