





"""simple docstring"""
import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin






class 					_SCREAMING_SNAKE_CASE(   A    ,	unittest.TestCase  ):
       SCREAMING_SNAKE_CASE_   :      Union[str, Any]							=  ProphetNetTokenizer
       SCREAMING_SNAKE_CASE_   :      Optional[int]							=  False




       def     _UpperCamelCase     (						self							)			->      Dict:





             """simple docstring"""






             super().setUp()

             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  [
                 '''[UNK]''',
                 '''[CLS]''',
                 '''[SEP]''',
                 '''[PAD]''',
                 '''[MASK]''',
                 '''want''',
                 '''##want''',
                 '''##ed''',
                 '''wa''',
                 '''un''',
                 '''runn''',
                 '''##ing''',
                 ''',''',
                 '''low''',
                 '''lowest''',
             ]
             __SCREAMING_SNAKE_CASE   :str						=  os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES['''vocab_file''']							)
             with open(self.vocab_file		,'''w'''		,encoding='''utf-8'''							) as vocab_writer:
                   vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      List[str]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  '''UNwant\u00E9d,running'''
             __SCREAMING_SNAKE_CASE   :str						=  '''unwanted, running'''
             return input_text, output_text




       def     _UpperCamelCase     (						self							)			->      Union[str, Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :int						=  self.tokenizer_class(self.vocab_file							)

             __SCREAMING_SNAKE_CASE   :Optional[int]						=  tokenizer.tokenize('''UNwant\u00E9d,running'''							)
             self.assertListEqual(SCREAMING_SNAKE_CASE__		,['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']							)
             self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__							)		,[9, 6, 7, 12, 10, 11]							)




       def     _UpperCamelCase     (						self							)			->      Optional[Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Tuple						=  BasicTokenizer()

             self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''							)		,['''ah''', '''\u535A''', '''\u63A8''', '''zz''']							)




       def     _UpperCamelCase     (						self							)			->      List[str]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Tuple						=  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__							)

             self.assertListEqual(
                 tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)		,['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
             self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)		,['''hello''']							)




       def     _UpperCamelCase     (						self							)			->      Optional[Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :str						=  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__		,strip_accents=SCREAMING_SNAKE_CASE__							)

             self.assertListEqual(
                 tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)		,['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
             self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)		,['''h\u00E9llo''']							)




       def     _UpperCamelCase     (						self							)			->      Any:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Tuple						=  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__		,strip_accents=SCREAMING_SNAKE_CASE__							)

             self.assertListEqual(
                 tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)		,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
             self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)		,['''hello''']							)




       def     _UpperCamelCase     (						self							)			->      List[Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__							)

             self.assertListEqual(
                 tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)		,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
             self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)		,['''hello''']							)




       def     _UpperCamelCase     (						self							)			->      str:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :int						=  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__							)

             self.assertListEqual(
                 tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)		,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




       def     _UpperCamelCase     (						self							)			->      str:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :int						=  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__		,strip_accents=SCREAMING_SNAKE_CASE__							)

             self.assertListEqual(
                 tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)		,['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




       def     _UpperCamelCase     (						self							)			->      Union[str, Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :List[str]						=  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__		,strip_accents=SCREAMING_SNAKE_CASE__							)

             self.assertListEqual(
                 tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)		,['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




       def     _UpperCamelCase     (						self							)			->      str:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Dict						=  BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__		,never_split=['''[UNK]''']							)

             self.assertListEqual(
                 tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''							)		,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']							)




       def     _UpperCamelCase     (						self							)			->      Dict:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Dict						=  ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

             __SCREAMING_SNAKE_CASE   :Any						=  {}
             for i, token in enumerate(SCREAMING_SNAKE_CASE__							):
                   __SCREAMING_SNAKE_CASE   :Optional[Any]						=  i
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  WordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE__		,unk_token='''[UNK]'''							)

             self.assertListEqual(tokenizer.tokenize(''''''							)		,[]							)

             self.assertListEqual(tokenizer.tokenize('''unwanted running'''							)		,['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']							)

             self.assertListEqual(tokenizer.tokenize('''unwantedX running'''							)		,['''[UNK]''', '''runn''', '''##ing''']							)




       @require_torch
       def     _UpperCamelCase     (						self							)			->      Optional[Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :List[Any]						=  self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''							)

             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
             __SCREAMING_SNAKE_CASE   :int						=  [10_37, 21_46, 2_04_23, 20_05, 76_80, 78_49, 39_89, 10_12, 1_02]
             __SCREAMING_SNAKE_CASE   :str						=  tokenizer(SCREAMING_SNAKE_CASE__		,padding=SCREAMING_SNAKE_CASE__		,return_tensors='''pt'''							)
             self.assertIsInstance(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  list(batch.input_ids.numpy()[0]							)
             self.assertListEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)

             self.assertEqual((2, 9)		,batch.input_ids.shape							)
             self.assertEqual((2, 9)		,batch.attention_mask.shape							)




       def     _UpperCamelCase     (						self							)			->      str:





             """simple docstring"""






             self.assertTrue(_is_whitespace(''' '''							)							)
             self.assertTrue(_is_whitespace('''\t'''							)							)
             self.assertTrue(_is_whitespace('''\r'''							)							)
             self.assertTrue(_is_whitespace('''\n'''							)							)
             self.assertTrue(_is_whitespace('''\u00A0'''							)							)

             self.assertFalse(_is_whitespace('''A'''							)							)
             self.assertFalse(_is_whitespace('''-'''							)							)




       def     _UpperCamelCase     (						self							)			->      List[str]:





             """simple docstring"""






             self.assertTrue(_is_control('''\u0005'''							)							)

             self.assertFalse(_is_control('''A'''							)							)
             self.assertFalse(_is_control(''' '''							)							)
             self.assertFalse(_is_control('''\t'''							)							)
             self.assertFalse(_is_control('''\r'''							)							)




       def     _UpperCamelCase     (						self							)			->      Optional[Any]:





             """simple docstring"""






             self.assertTrue(_is_punctuation('''-'''							)							)
             self.assertTrue(_is_punctuation('''$'''							)							)
             self.assertTrue(_is_punctuation('''`'''							)							)
             self.assertTrue(_is_punctuation('''.'''							)							)

             self.assertFalse(_is_punctuation('''A'''							)							)
             self.assertFalse(_is_punctuation(''' '''							)							)




       @slow
       def     _UpperCamelCase     (						self							)			->      List[Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :List[Any]						=  self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased'''							)

             __SCREAMING_SNAKE_CASE   :List[Any]						=  tokenizer.encode('''sequence builders'''		,add_special_tokens=SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :List[Any]						=  tokenizer.encode('''multi-sequence build'''		,add_special_tokens=SCREAMING_SNAKE_CASE__							)

             __SCREAMING_SNAKE_CASE   :str						=  tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE   :Any						=  tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)

             assert encoded_sentence == text + [1_02]
             assert encoded_pair == text + [1_02] + text_a + [1_02]






"""simple docstring"""
from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCamelCase_								=		logging.get_logger(__name__)

lowerCamelCase_								=		{
    "microsoft/beit-base-patch16-224-pt22k": (
        "https://huggingface.co/microsoft/beit-base-patch16-224-pt22k/resolve/main/config.json"
    ),
    # See all BEiT models at https://huggingface.co/models?filter=beit
}






class 					_SCREAMING_SNAKE_CASE(   A  ):
       SCREAMING_SNAKE_CASE_   :      str							=  '''beit'''




       def __init__(						self		,SCREAMING_SNAKE_CASE__=81_92		,SCREAMING_SNAKE_CASE__=7_68		,SCREAMING_SNAKE_CASE__=12		,SCREAMING_SNAKE_CASE__=12		,SCREAMING_SNAKE_CASE__=30_72		,SCREAMING_SNAKE_CASE__="gelu"		,SCREAMING_SNAKE_CASE__=0.0		,SCREAMING_SNAKE_CASE__=0.0		,SCREAMING_SNAKE_CASE__=0.0_2		,SCREAMING_SNAKE_CASE__=1E-12		,SCREAMING_SNAKE_CASE__=2_24		,SCREAMING_SNAKE_CASE__=16		,SCREAMING_SNAKE_CASE__=3		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=0.1		,SCREAMING_SNAKE_CASE__=0.1		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=[3, 5, 7, 11]		,SCREAMING_SNAKE_CASE__=[1, 2, 3, 6]		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=0.4		,SCREAMING_SNAKE_CASE__=2_56		,SCREAMING_SNAKE_CASE__=1		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=2_55		,**SCREAMING_SNAKE_CASE__		,)			->      Optional[Any]:





             """simple docstring"""






             super().__init__(**SCREAMING_SNAKE_CASE__							)

             __SCREAMING_SNAKE_CASE   :Any						=  vocab_size
             __SCREAMING_SNAKE_CASE   :Dict						=  hidden_size
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  num_hidden_layers
             __SCREAMING_SNAKE_CASE   :List[str]						=  num_attention_heads
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  intermediate_size
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  hidden_act
             __SCREAMING_SNAKE_CASE   :Tuple						=  hidden_dropout_prob
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  attention_probs_dropout_prob
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  initializer_range
             __SCREAMING_SNAKE_CASE   :str						=  layer_norm_eps

             __SCREAMING_SNAKE_CASE   :int						=  image_size
             __SCREAMING_SNAKE_CASE   :Tuple						=  patch_size
             __SCREAMING_SNAKE_CASE   :Any						=  num_channels
             __SCREAMING_SNAKE_CASE   :Any						=  use_mask_token
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  use_absolute_position_embeddings
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  use_relative_position_bias
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  use_shared_relative_position_bias
             __SCREAMING_SNAKE_CASE   :List[str]						=  layer_scale_init_value
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  drop_path_rate
             __SCREAMING_SNAKE_CASE   :str						=  use_mean_pooling
             # decode head attributes (semantic segmentation)
             __SCREAMING_SNAKE_CASE   :Dict						=  out_indices
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  pool_scales
             # auxiliary head attributes (semantic segmentation)
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  use_auxiliary_head
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  auxiliary_loss_weight
             __SCREAMING_SNAKE_CASE   :Dict						=  auxiliary_channels
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  auxiliary_num_convs
             __SCREAMING_SNAKE_CASE   :List[str]						=  auxiliary_concat_input
             __SCREAMING_SNAKE_CASE   :List[Any]						=  semantic_loss_ignore_index






class 					_SCREAMING_SNAKE_CASE(   A  ):
       SCREAMING_SNAKE_CASE_   :      List[Any]							=  version.parse('''1.11'''  )




       @property
       def     _UpperCamelCase     (						self							)			->      Mapping[str, Mapping[int, str]]:





             """simple docstring"""






             return OrderedDict(
                 [
                     ('''pixel_values''', {0: '''batch''', 1: '''num_channels''', 2: '''height''', 3: '''width'''}),
                 ]							)




       @property
       def     _UpperCamelCase     (						self							)			->      float:





             """simple docstring"""






             return 1E-4