


import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin







@custom_tokenizers
class 			_lowercase (							_UpperCAmelCase							,  unittest.TestCase):


       """simple docstring"""




       A__											=			BertJapaneseTokenizer
       A__											=			False
       A__											=			True





       def 						lowerCAmelCase     (   self							:			str       ):


             '''simple docstring'''



             super().setUp()

             lowerCamelCase__		:				Optional[Any]    =  [
                 "[UNK]",
                 "[CLS]",
                 "[SEP]",
                 "こんにちは",
                 "こん",
                 "にちは",
                 "ばんは",
                 "##こん",
                 "##にちは",
                 "##ばんは",
                 "世界",
                 "##世界",
                 "、",
                 "##、",
                 "。",
                 "##。",
             ]

             lowerCamelCase__		:				int    =  os.path.join(self.tmpdirname					,      VOCAB_FILES_NAMES["vocab_file"]       )
             with open(self.vocab_file					,      "w"					,      encoding="utf-8"       ) as vocab_writer:
                   vocab_writer.write("".join([x + "\n" for x in vocab_tokens]       )       )





       def 						lowerCAmelCase     (   self							:			Tuple					,      __lowerCamelCase							:			Optional[int]       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[Any]    =  "こんにちは、世界。 \nこんばんは、世界。"
             lowerCamelCase__		:				Dict    =  "こんにちは 、 世界 。 こんばんは 、 世界 。"
             return input_text, output_text





       def 						lowerCAmelCase     (   self							:			Tuple					,      __lowerCamelCase							:			Union[str, Any]       ):


             '''simple docstring'''



             lowerCamelCase__    ,   lowerCamelCase__		:				List[str]    =  self.get_input_output_texts(lowerCAmelCase__       )
             lowerCamelCase__		:				Optional[int]    =  tokenizer.encode(lowerCAmelCase__					,      add_special_tokens=lowerCAmelCase__       )
             lowerCamelCase__		:				int    =  tokenizer.decode(lowerCAmelCase__					,      clean_up_tokenization_spaces=lowerCAmelCase__       )
             return text, ids





       def 						lowerCAmelCase     (   self							:			Dict       ):


             '''simple docstring'''



             pass  # TODO add if relevant





       def 						lowerCAmelCase     (   self							:			Optional[Any]       ):


             '''simple docstring'''



             pass  # TODO add if relevant





       def 						lowerCAmelCase     (   self							:			Any       ):


             '''simple docstring'''



             pass  # TODO add if relevant





       def 						lowerCAmelCase     (   self							:			Tuple       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[Any]    =  self.tokenizer_class(self.vocab_file       )

             lowerCamelCase__		:				int    =  tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"       )
             self.assertListEqual(lowerCAmelCase__					,      ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]       )
             self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__       )					,      [3, 12, 10, 14, 4, 9, 12, 10, 14]       )





       def 						lowerCAmelCase     (   self							:			Tuple       ):


             '''simple docstring'''



             lowerCamelCase__		:				Union[str, Any]    =  self.tokenizer_class(self.vocab_file					,      word_tokenizer_type="mecab"       )
             self.assertIsNotNone(lowerCAmelCase__       )

             lowerCamelCase__		:				int    =  "こんにちは、世界。\nこんばんは、世界。"
             lowerCamelCase__		:				Dict    =  tokenizer.tokenize(lowerCAmelCase__       )
             self.assertListEqual(lowerCAmelCase__					,      ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]       )
             self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__       )					,      [3, 12, 10, 14, 4, 9, 12, 10, 14]       )

             lowerCamelCase__		:				Dict    =  os.path.join(self.tmpdirname					,      "tokenizer.bin"       )
             with open(lowerCAmelCase__					,      "wb"       ) as handle:
                   pickle.dump(lowerCAmelCase__					,      lowerCAmelCase__       )

             with open(lowerCAmelCase__					,      "rb"       ) as handle:
                   lowerCamelCase__		:				Any    =  pickle.load(lowerCAmelCase__       )

             lowerCamelCase__		:				Any    =  tokenizer_new.tokenize(lowerCAmelCase__       )

             self.assertListEqual(lowerCAmelCase__					,      lowerCAmelCase__       )





       def 						lowerCAmelCase     (   self							:			Dict       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[str]    =  MecabTokenizer(mecab_dic="ipadic"       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "       )					,      ["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]					,      )





       def 						lowerCAmelCase     (   self							:			int       ):


             '''simple docstring'''



             try:
                   lowerCamelCase__		:				List[Any]    =  MecabTokenizer(mecab_dic="unidic_lite"       )
             except ModuleNotFoundError:
                   return

             self.assertListEqual(
                 tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "       )					,      ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]					,      )





       def 						lowerCAmelCase     (   self							:			Any       ):


             '''simple docstring'''



             try:
                   lowerCamelCase__		:				List[Any]    =  MecabTokenizer(mecab_dic="unidic"       )
             except ModuleNotFoundError:
                   return

             self.assertListEqual(
                 tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "       )					,      ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]					,      )





       def 						lowerCAmelCase     (   self							:			Dict       ):


             '''simple docstring'''



             lowerCamelCase__		:				str    =  MecabTokenizer(do_lower_case=lowerCAmelCase__					,      mecab_dic="ipadic"       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "       )					,      ["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]					,      )





       def 						lowerCAmelCase     (   self							:			Union[str, Any]       ):


             '''simple docstring'''



             try:
                   lowerCamelCase__		:				Any    =  MecabTokenizer(
                       do_lower_case=lowerCAmelCase__					,      normalize_text=lowerCAmelCase__					,      mecab_option="-d /usr/local/lib/mecab/dic/jumandic"       )
             except RuntimeError:
                   # if dict doesn't exist in the system, previous code raises this error.
                   return

             self.assertListEqual(
                 tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "       )					,      ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]					,      )





       def 						lowerCAmelCase     (   self							:			Tuple       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[str]    =  MecabTokenizer(normalize_text=lowerCAmelCase__					,      mecab_dic="ipadic"       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "       )					,      ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]					,      )





       @require_sudachi
       def 						lowerCAmelCase     (   self							:			int       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[Any]    =  self.tokenizer_class(self.vocab_file					,      word_tokenizer_type="sudachi"       )
             self.assertIsNotNone(lowerCAmelCase__       )

             lowerCamelCase__		:				Optional[Any]    =  "こんにちは、世界。\nこんばんは、世界。"
             lowerCamelCase__		:				str    =  tokenizer.tokenize(lowerCAmelCase__       )
             self.assertListEqual(lowerCAmelCase__					,      ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]       )
             self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__       )					,      [3, 12, 10, 14, 4, 9, 12, 10, 14]       )

             lowerCamelCase__		:				Union[str, Any]    =  os.path.join(self.tmpdirname					,      "tokenizer.bin"       )
             with open(lowerCAmelCase__					,      "wb"       ) as handle:
                   pickle.dump(lowerCAmelCase__					,      lowerCAmelCase__       )

             with open(lowerCAmelCase__					,      "rb"       ) as handle:
                   lowerCamelCase__		:				Union[str, Any]    =  pickle.load(lowerCAmelCase__       )

             lowerCamelCase__		:				Optional[Any]    =  tokenizer_new.tokenize(lowerCAmelCase__       )

             self.assertListEqual(lowerCAmelCase__					,      lowerCAmelCase__       )





       @require_sudachi
       def 						lowerCAmelCase     (   self							:			Optional[int]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Dict    =  SudachiTokenizer(sudachi_dict_type="core"       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "       )					,      [" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]					,      )





       @require_sudachi
       def 						lowerCAmelCase     (   self							:			Any       ):


             '''simple docstring'''



             lowerCamelCase__		:				Tuple    =  SudachiTokenizer(sudachi_dict_type="core"					,      sudachi_split_mode="A"       )

             self.assertListEqual(tokenizer.tokenize("外国人参政権"       )					,      ["外国", "人", "参政", "権"]       )





       @require_sudachi
       def 						lowerCAmelCase     (   self							:			List[Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[int]    =  SudachiTokenizer(sudachi_dict_type="core"					,      sudachi_split_mode="B"       )

             self.assertListEqual(tokenizer.tokenize("外国人参政権"       )					,      ["外国人", "参政権"]       )





       @require_sudachi
       def 						lowerCAmelCase     (   self							:			Tuple       ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[int]    =  SudachiTokenizer(sudachi_dict_type="core"					,      sudachi_split_mode="C"       )

             self.assertListEqual(tokenizer.tokenize("外国人参政権"       )					,      ["外国人参政権"]       )





       @require_sudachi
       def 						lowerCAmelCase     (   self							:			Dict       ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[Any]    =  SudachiTokenizer(do_lower_case=lowerCAmelCase__					,      sudachi_dict_type="core"       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "       )					,      [" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]					,      )





       @require_sudachi
       def 						lowerCAmelCase     (   self							:			str       ):


             '''simple docstring'''



             lowerCamelCase__		:				Union[str, Any]    =  SudachiTokenizer(normalize_text=lowerCAmelCase__					,      sudachi_dict_type="core"       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "       )					,      [" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]					,      )





       @require_sudachi
       def 						lowerCAmelCase     (   self							:			Tuple       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[Any]    =  SudachiTokenizer(trim_whitespace=lowerCAmelCase__					,      sudachi_dict_type="core"       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "       )					,      ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]					,      )





       @require_jumanpp
       def 						lowerCAmelCase     (   self							:			List[Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				str    =  self.tokenizer_class(self.vocab_file					,      word_tokenizer_type="jumanpp"       )
             self.assertIsNotNone(lowerCAmelCase__       )

             lowerCamelCase__		:				Optional[Any]    =  "こんにちは、世界。\nこんばんは、世界。"
             lowerCamelCase__		:				Tuple    =  tokenizer.tokenize(lowerCAmelCase__       )
             self.assertListEqual(lowerCAmelCase__					,      ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]       )
             self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__       )					,      [3, 12, 10, 14, 4, 9, 12, 10, 14]       )

             lowerCamelCase__		:				List[str]    =  os.path.join(self.tmpdirname					,      "tokenizer.bin"       )
             with open(lowerCAmelCase__					,      "wb"       ) as handle:
                   pickle.dump(lowerCAmelCase__					,      lowerCAmelCase__       )

             with open(lowerCAmelCase__					,      "rb"       ) as handle:
                   lowerCamelCase__		:				Any    =  pickle.load(lowerCAmelCase__       )

             lowerCamelCase__		:				List[Any]    =  tokenizer_new.tokenize(lowerCAmelCase__       )

             self.assertListEqual(lowerCAmelCase__					,      lowerCAmelCase__       )





       @require_jumanpp
       def 						lowerCAmelCase     (   self							:			List[Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[str]    =  JumanppTokenizer()

             self.assertListEqual(
                 tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "       )					,      ["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]					,      )





       @require_jumanpp
       def 						lowerCAmelCase     (   self							:			Optional[int]       ):


             '''simple docstring'''



             lowerCamelCase__		:				int    =  JumanppTokenizer(do_lower_case=lowerCAmelCase__       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "       )					,      ["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]					,      )





       @require_jumanpp
       def 						lowerCAmelCase     (   self							:			Tuple       ):


             '''simple docstring'''



             lowerCamelCase__		:				str    =  JumanppTokenizer(normalize_text=lowerCAmelCase__       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "       )					,      ["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]					,      )





       @require_jumanpp
       def 						lowerCAmelCase     (   self							:			Dict       ):


             '''simple docstring'''



             lowerCamelCase__		:				Any    =  JumanppTokenizer(trim_whitespace=lowerCAmelCase__       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "       )					,      ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]					,      )





       @require_jumanpp
       def 						lowerCAmelCase     (   self							:			str       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[str]    =  JumanppTokenizer()

             self.assertListEqual(
                 tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"       )					,      ["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]					,      )





       def 						lowerCAmelCase     (   self							:			Dict       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[str]    =  ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

             lowerCamelCase__		:				Dict    =  {}
             for i, token in enumerate(lowerCAmelCase__       ):
                   lowerCamelCase__		:				Optional[Any]    =  i
             lowerCamelCase__		:				Dict    =  WordpieceTokenizer(vocab=lowerCAmelCase__					,      unk_token="[UNK]"       )

             self.assertListEqual(tokenizer.tokenize(""       )					,      []       )

             self.assertListEqual(tokenizer.tokenize("こんにちは"       )					,      ["こんにちは"]       )

             self.assertListEqual(tokenizer.tokenize("こんばんは"       )					,      ["こん", "##ばんは"]       )

             self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"       )					,      ["こん", "##ばんは", "[UNK]", "こんにちは"]       )





       def 						lowerCAmelCase     (   self							:			Optional[Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Tuple    =  BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"       )
             lowerCamelCase__		:				Optional[Any]    =  tokenizer.subword_tokenizer

             lowerCamelCase__		:				List[Any]    =  subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"       )
             self.assertListEqual(lowerCAmelCase__					,      ["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]       )

             lowerCamelCase__		:				List[Any]    =  subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"       )
             self.assertListEqual(lowerCAmelCase__					,      ["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]       )





       def 						lowerCAmelCase     (   self							:			Any       ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[Any]    =  self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"       )

             lowerCamelCase__		:				str    =  tokenizer.encode("ありがとう。"					,      add_special_tokens=lowerCAmelCase__       )
             lowerCamelCase__		:				List[Any]    =  tokenizer.encode("どういたしまして。"					,      add_special_tokens=lowerCAmelCase__       )

             lowerCamelCase__		:				int    =  tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__       )
             lowerCamelCase__		:				List[str]    =  tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__					,      lowerCAmelCase__       )

             # 2 is for "[CLS]", 3 is for "[SEP]"
             assert encoded_sentence == [2] + text + [3]
             assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class 			_lowercase (							_UpperCAmelCase							,  unittest.TestCase):


       """simple docstring"""




       A__											=			BertJapaneseTokenizer
       A__											=			False





       def 						lowerCAmelCase     (   self							:			Dict       ):


             '''simple docstring'''



             super().setUp()

             lowerCamelCase__		:				Optional[Any]    =  ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

             lowerCamelCase__		:				int    =  os.path.join(self.tmpdirname					,      VOCAB_FILES_NAMES["vocab_file"]       )
             with open(self.vocab_file					,      "w"					,      encoding="utf-8"       ) as vocab_writer:
                   vocab_writer.write("".join([x + "\n" for x in vocab_tokens]       )       )





       def 						lowerCAmelCase     (   self							:			Optional[int]					,      **__lowerCamelCase							:			Any       ):


             '''simple docstring'''



             return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,      subword_tokenizer_type="character"					,      **lowerCAmelCase__       )





       def 						lowerCAmelCase     (   self							:			Union[str, Any]					,      __lowerCamelCase							:			Dict       ):


             '''simple docstring'''



             lowerCamelCase__		:				str    =  "こんにちは、世界。 \nこんばんは、世界。"
             lowerCamelCase__		:				Dict    =  "こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
             return input_text, output_text





       def 						lowerCAmelCase     (   self							:			Dict       ):


             '''simple docstring'''



             pass  # TODO add if relevant





       def 						lowerCAmelCase     (   self							:			Optional[Any]       ):


             '''simple docstring'''



             pass  # TODO add if relevant





       def 						lowerCAmelCase     (   self							:			int       ):


             '''simple docstring'''



             pass  # TODO add if relevant





       def 						lowerCAmelCase     (   self							:			str       ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[int]    =  self.tokenizer_class(self.vocab_file					,      subword_tokenizer_type="character"       )

             lowerCamelCase__		:				int    =  tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"       )
             self.assertListEqual(
                 lowerCAmelCase__					,      ["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]       )
             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(lowerCAmelCase__       )					,      [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]       )





       def 						lowerCAmelCase     (   self							:			Any       ):


             '''simple docstring'''



             lowerCamelCase__		:				Any    =  ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

             lowerCamelCase__		:				Optional[int]    =  {}
             for i, token in enumerate(lowerCAmelCase__       ):
                   lowerCamelCase__		:				Optional[int]    =  i
             lowerCamelCase__		:				int    =  CharacterTokenizer(vocab=lowerCAmelCase__					,      unk_token="[UNK]"       )

             self.assertListEqual(tokenizer.tokenize(""       )					,      []       )

             self.assertListEqual(tokenizer.tokenize("こんにちは"       )					,      ["こ", "ん", "に", "ち", "は"]       )

             self.assertListEqual(tokenizer.tokenize("こんにちほ"       )					,      ["こ", "ん", "に", "ち", "[UNK]"]       )





       def 						lowerCAmelCase     (   self							:			Union[str, Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Tuple    =  self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"       )

             lowerCamelCase__		:				Dict    =  tokenizer.encode("ありがとう。"					,      add_special_tokens=lowerCAmelCase__       )
             lowerCamelCase__		:				Union[str, Any]    =  tokenizer.encode("どういたしまして。"					,      add_special_tokens=lowerCAmelCase__       )

             lowerCamelCase__		:				List[Any]    =  tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__       )
             lowerCamelCase__		:				Dict    =  tokenizer.build_inputs_with_special_tokens(lowerCAmelCase__					,      lowerCAmelCase__       )

             # 2 is for "[CLS]", 3 is for "[SEP]"
             assert encoded_sentence == [2] + text + [3]
             assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class 			_lowercase (							unittest.TestCase):


       """simple docstring"""




       def 						lowerCAmelCase     (   self							:			Union[str, Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Dict    =  "cl-tohoku/bert-base-japanese"
             lowerCamelCase__		:				List[Any]    =  AutoTokenizer.from_pretrained(lowerCAmelCase__       )
             self.assertIsInstance(lowerCAmelCase__					,      lowerCAmelCase__       )







class 			_lowercase (							unittest.TestCase):


       """simple docstring"""




       def 						lowerCAmelCase     (   self							:			Optional[int]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Dict    =  "cl-tohoku/bert-base-japanese"
             with self.assertLogs("transformers"					,      level="WARNING"       ) as cm:
                   BertTokenizer.from_pretrained(lowerCAmelCase__       )
                   self.assertTrue(
                       cm.records[0].message.startswith(
                           "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                           " is called from."       )       )
             lowerCamelCase__		:				Tuple    =  "bert-base-cased"
             with self.assertLogs("transformers"					,      level="WARNING"       ) as cm:
                   BertJapaneseTokenizer.from_pretrained(lowerCAmelCase__       )
                   self.assertTrue(
                       cm.records[0].message.startswith(
                           "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                           " is called from."       )       )



import argparse
import gc
import json
import os

import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


__a  :     str       =  1_6
__a  :     str       =  3_2





def 				UpperCAmelCase						(    lowercase							):







				"""simple docstring"""



				return int(x / 2**20							)




class    _UpperCamelCase      :


			"""simple docstring"""






			def __enter__(   self						)     ->					str:

							'''simple docstring'''


							gc.collect()
							torch.cuda.empty_cache()
							torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
							__lowercase 			=							torch.cuda.memory_allocated()
							return self





			def __exit__(   self      ,   *lowerCAmelCase__						)     ->					int:

							'''simple docstring'''


							gc.collect()
							torch.cuda.empty_cache()
							__lowercase 			=							torch.cuda.memory_allocated()
							__lowercase 			=							torch.cuda.max_memory_allocated()
							__lowercase 			=							bamb(self.end - self.begin						)
							__lowercase 			=							bamb(self.peak - self.begin						)
							# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")





def 				UpperCAmelCase						(    lowercase							,			lowercase = 16							,			lowercase = "bert-base-cased"							,			lowercase = 320							,			lowercase = 160							,			):







				"""simple docstring"""



				__lowercase 			=							AutoTokenizer.from_pretrained(lowercase							)
				__lowercase 			=							load_dataset(
				    '''glue'''							,			'''mrpc'''							,			split={'''train''': F"train[:{n_train}]", '''validation''': F"validation[:{n_val}]"}							)

				def tokenize_function(lowercase							):
								# max_length=None => use the model max length (it's actually the default)
								__lowercase 			=							tokenizer(examples['''sentence1''']							,			examples['''sentence2''']							,			truncation=lowercase							,			max_length=lowercase							)
								return outputs

				# Apply the method we just defined to all the examples in all the splits of the dataset
				__lowercase 			=							datasets.map(
				    lowercase							,			batched=lowercase							,			remove_columns=['''idx''', '''sentence1''', '''sentence2''']							,			load_from_cache_file=lowercase							)

				# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
				# transformers library
				__lowercase 			=							tokenized_datasets.rename_column('''label'''							,			'''labels'''							)

				def collate_fn(lowercase							):
								# On TPU it's best to pad everything to the same length or training will be very slow.
								if accelerator.distributed_type == DistributedType.TPU:
												return tokenizer.pad(lowercase							,			padding='''max_length'''							,			max_length=128							,			return_tensors='''pt'''							)
								return tokenizer.pad(lowercase							,			padding='''longest'''							,			return_tensors='''pt'''							)

				# Instantiate dataloaders.
				__lowercase 			=							DataLoader(
				    tokenized_datasets['''train''']							,			shuffle=lowercase							,			collate_fn=lowercase							,			batch_size=lowercase							)
				__lowercase 			=							DataLoader(
				    tokenized_datasets['''validation''']							,			shuffle=lowercase							,			collate_fn=lowercase							,			batch_size=lowercase							)

				return train_dataloader, eval_dataloader





def 				UpperCAmelCase						(    lowercase							,			lowercase							):







				"""simple docstring"""



				__lowercase 			=							Accelerator()

				# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
				__lowercase 			=							config['''lr''']
				__lowercase 			=							int(config['''num_epochs''']							)
				__lowercase 			=							int(config['''seed''']							)
				__lowercase 			=							int(config['''batch_size''']							)
				__lowercase 			=							args.model_name_or_path

				set_seed(lowercase							)
				__lowercase			,     __lowercase 			=							get_dataloaders(lowercase							,			lowercase							,			lowercase							,			args.n_train							,			args.n_val							)

				# Instantiate the model (we build the model here so that the seed also control new weights initialization)
				__lowercase 			=							AutoModelForSequenceClassification.from_pretrained(lowercase							,			return_dict=lowercase							)

				# Instantiate optimizer
				__lowercase 			=							(
				    AdamW
				    if accelerator.state.deepspeed_plugin is None
				    or '''optimizer''' not in accelerator.state.deepspeed_plugin.deepspeed_config
				    else DummyOptim
				)
				__lowercase 			=							optimizer_cls(params=model.parameters()							,			lr=lowercase							)

				if accelerator.state.deepspeed_plugin is not None:
								__lowercase 			=							accelerator.state.deepspeed_plugin.deepspeed_config[
								    '''gradient_accumulation_steps'''
								]
				else:
								__lowercase 			=							1
				__lowercase 			=							(len(lowercase							) * num_epochs) // gradient_accumulation_steps

				# Instantiate scheduler
				if (
				    accelerator.state.deepspeed_plugin is None
				    or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
				):
								__lowercase 			=							get_linear_schedule_with_warmup(
								    optimizer=lowercase							,			num_warmup_steps=0							,			num_training_steps=lowercase							,			)
				else:
								__lowercase 			=							DummyScheduler(lowercase							,			total_num_steps=lowercase							,			warmup_num_steps=0							)

				# Prepare everything
				# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
				# prepare method.
				__lowercase			,     __lowercase			,     __lowercase			,     __lowercase			,     __lowercase 			=							accelerator.prepare(
				    lowercase							,			lowercase							,			lowercase							,			lowercase							,			lowercase							)

				# We need to keep track of how many total steps we have iterated over
				__lowercase 			=							0
				# We also need to keep track of the stating epoch so files are named properly
				__lowercase 			=							0

				# Now we train the model
				__lowercase 			=							{}
				for epoch in range(lowercase							,			lowercase							):
								with TorchTracemalloc() as tracemalloc:
												model.train()
												for step, batch in enumerate(lowercase							):
																__lowercase 			=							model(**lowercase							)
																__lowercase 			=							outputs.loss
																__lowercase 			=							loss / gradient_accumulation_steps
																accelerator.backward(lowercase							)
																if step % gradient_accumulation_steps == 0:
																				optimizer.step()
																				lr_scheduler.step()
																				optimizer.zero_grad()

																overall_step += 1

        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
								accelerator.print('''Memory before entering the train : {}'''.format(bamb(tracemalloc.begin							)							)							)
								accelerator.print('''Memory consumed at the end of the train (end-begin): {}'''.format(tracemalloc.used							)							)
								accelerator.print('''Peak Memory consumed during the train (max-begin): {}'''.format(tracemalloc.peaked							)							)
								accelerator.print(
								    '''Total Peak Memory consumed during the train (max): {}'''.format(
								        tracemalloc.peaked + bamb(tracemalloc.begin							)							)							)
								__lowercase 			=							tracemalloc.peaked + bamb(tracemalloc.begin							)
								if args.peak_memory_upper_bound is not None:
												assert (
												    train_total_peak_memory[F"epoch-{epoch}"] <= args.peak_memory_upper_bound
												), "Peak memory usage exceeded the upper bound"

				accelerator.wait_for_everyone()
				if accelerator.is_main_process:
								with open(os.path.join(args.output_dir							,			'''peak_memory_utilization.json'''							)							,			'''w'''							) as f:
												json.dump(lowercase							,			lowercase							)





def 				UpperCAmelCase						(    ):







				"""simple docstring"""



				__lowercase 			=							argparse.ArgumentParser(description='''Simple example of training script tracking peak GPU memory usage.'''							)
				parser.add_argument(
				    '''--model_name_or_path'''							,			type=lowercase							,			default='''bert-base-cased'''							,			help='''Path to pretrained model or model identifier from huggingface.co/models.'''							,			required=lowercase							,			)
				parser.add_argument(
				    '''--output_dir'''							,			type=lowercase							,			default='''.'''							,			help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'''							,			)
				parser.add_argument(
				    '''--peak_memory_upper_bound'''							,			type=lowercase							,			default=lowercase							,			help='''The upper bound of peak memory usage in MB. If set, the training will throw an error if the peak memory usage exceeds this value.'''							,			)
				parser.add_argument(
				    '''--n_train'''							,			type=lowercase							,			default=320							,			help='''Number of training examples to use.'''							,			)
				parser.add_argument(
				    '''--n_val'''							,			type=lowercase							,			default=160							,			help='''Number of validation examples to use.'''							,			)
				parser.add_argument(
				    '''--num_epochs'''							,			type=lowercase							,			default=1							,			help='''Number of train epochs.'''							,			)
				__lowercase 			=							parser.parse_args()
				__lowercase 			=							{'''lr''': 2E-5, '''num_epochs''': args.num_epochs, '''seed''': 42, '''batch_size''': 16}
				training_function(lowercase							,			lowercase							)


if __name__ == "__main__":
		main()