
'''simple docstring'''







import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin


class 						__UpperCAmelCase      (					_lowerCamelCase						,   unittest.TestCase						):
							__lowercase      		=       ProphetNetTokenizer
							__lowercase      		=       False


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								super().setUp()

								_snake_case   						=		[
								    '''[UNK]''',
								    '''[CLS]''',
								    '''[SEP]''',
								    '''[PAD]''',
								    '''[MASK]''',
								    '''want''',
								    '''##want''',
								    '''##ed''',
								    '''wa''',
								    '''un''',
								    '''runn''',
								    '''##ing''',
								    ''',''',
								    '''low''',
								    '''lowest''',
								]
								_snake_case   						=		os.path.join(self.tmpdirname    ,   VOCAB_FILES_NAMES['vocab_file']  )
								with open(self.vocab_file    ,   'w'    ,   encoding='utf-8'  ) as vocab_writer:
									vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )


							def  lowerCamelCase				(       self    ,   lowerCAmelCase_  ):





								"""simple docstring"""






								_snake_case   						=		'''UNwant\u00E9d,running'''
								_snake_case   						=		'''unwanted, running'''
								return input_text, output_text


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.tokenizer_class(self.vocab_file  )

								_snake_case   						=		tokenizer.tokenize('UNwant\u00E9d,running'  )
								self.assertListEqual(SCREAMING_SNAKE_CASE__    ,   ['un', '##want', '##ed', ',', 'runn', '##ing']  )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__  )    ,   [9, 6, 7, 12, 10, 11]  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )    ,   ['ah', '\u535A', '\u63A8', 'zz']  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )    ,   ['hello', '!', 'how', 'are', 'you', '?']  )
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,   ['hello']  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__    ,   strip_accents=SCREAMING_SNAKE_CASE__  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,   ['hällo', '!', 'how', 'are', 'you', '?']  )
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,   ['h\u00E9llo']  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__    ,   strip_accents=SCREAMING_SNAKE_CASE__  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,   ['hallo', '!', 'how', 'are', 'you', '?']  )
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,   ['hello']  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,   ['hallo', '!', 'how', 'are', 'you', '?']  )
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )    ,   ['hello']  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )    ,   ['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__    ,   strip_accents=SCREAMING_SNAKE_CASE__  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,   ['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__    ,   strip_accents=SCREAMING_SNAKE_CASE__  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )    ,   ['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE__    ,   never_split=['[UNK]']  )

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )    ,   ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

								_snake_case   						=		{}
								for i, token in enumerate(SCREAMING_SNAKE_CASE__  ):
									_snake_case   						=		i
								_snake_case   						=		WordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE__    ,   unk_token='[UNK]'  )

								self.assertListEqual(tokenizer.tokenize(''  )    ,   []  )

								self.assertListEqual(tokenizer.tokenize('unwanted running'  )    ,   ['un', '##want', '##ed', 'runn', '##ing']  )

								self.assertListEqual(tokenizer.tokenize('unwantedX running'  )    ,   ['[UNK]', 'runn', '##ing']  )


							@require_torch
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'  )

								_snake_case   						=		['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
								_snake_case   						=		[10_37, 21_46, 2_04_23, 20_05, 76_80, 78_49, 39_89, 10_12, 1_02]
								_snake_case   						=		tokenizer(SCREAMING_SNAKE_CASE__    ,   padding=SCREAMING_SNAKE_CASE__    ,   return_tensors='pt'  )
								self.assertIsInstance(SCREAMING_SNAKE_CASE__    ,   SCREAMING_SNAKE_CASE__  )
								_snake_case   						=		list(batch.input_ids.numpy()[0]  )
								self.assertListEqual(SCREAMING_SNAKE_CASE__    ,   SCREAMING_SNAKE_CASE__  )

								self.assertEqual((2, 9)    ,   batch.input_ids.shape  )
								self.assertEqual((2, 9)    ,   batch.attention_mask.shape  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								self.assertTrue(_is_whitespace(' '  )  )
								self.assertTrue(_is_whitespace('\t'  )  )
								self.assertTrue(_is_whitespace('\r'  )  )
								self.assertTrue(_is_whitespace('\n'  )  )
								self.assertTrue(_is_whitespace('\u00A0'  )  )

								self.assertFalse(_is_whitespace('A'  )  )
								self.assertFalse(_is_whitespace('-'  )  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								self.assertTrue(_is_control('\u0005'  )  )

								self.assertFalse(_is_control('A'  )  )
								self.assertFalse(_is_control(' '  )  )
								self.assertFalse(_is_control('\t'  )  )
								self.assertFalse(_is_control('\r'  )  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								self.assertTrue(_is_punctuation('-'  )  )
								self.assertTrue(_is_punctuation('$'  )  )
								self.assertTrue(_is_punctuation('`'  )  )
								self.assertTrue(_is_punctuation('.'  )  )

								self.assertFalse(_is_punctuation('A'  )  )
								self.assertFalse(_is_punctuation(' '  )  )




							@slow
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'  )

								_snake_case   						=		tokenizer.encode('sequence builders'    ,   add_special_tokens=SCREAMING_SNAKE_CASE__  )
								_snake_case   						=		tokenizer.encode('multi-sequence build'    ,   add_special_tokens=SCREAMING_SNAKE_CASE__  )

								_snake_case   						=		tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE__  )
								_snake_case   						=		tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE__    ,   SCREAMING_SNAKE_CASE__  )

								assert encoded_sentence == text + [1_02]
								assert encoded_pair == text + [1_02] + text_a + [1_02]








"""simple docstring"""
from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCamelCase_								=		logging.get_logger(__name__)

lowerCamelCase_								=		{
    "microsoft/beit-base-patch16-224-pt22k": (
        "https://huggingface.co/microsoft/beit-base-patch16-224-pt22k/resolve/main/config.json"
    ),
    # See all BEiT models at https://huggingface.co/models?filter=beit
}






class 					_SCREAMING_SNAKE_CASE(   A  ):
       SCREAMING_SNAKE_CASE_   :      str							=  '''beit'''




       def __init__(						self		,SCREAMING_SNAKE_CASE__=81_92		,SCREAMING_SNAKE_CASE__=7_68		,SCREAMING_SNAKE_CASE__=12		,SCREAMING_SNAKE_CASE__=12		,SCREAMING_SNAKE_CASE__=30_72		,SCREAMING_SNAKE_CASE__="gelu"		,SCREAMING_SNAKE_CASE__=0.0		,SCREAMING_SNAKE_CASE__=0.0		,SCREAMING_SNAKE_CASE__=0.0_2		,SCREAMING_SNAKE_CASE__=1E-12		,SCREAMING_SNAKE_CASE__=2_24		,SCREAMING_SNAKE_CASE__=16		,SCREAMING_SNAKE_CASE__=3		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=0.1		,SCREAMING_SNAKE_CASE__=0.1		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=[3, 5, 7, 11]		,SCREAMING_SNAKE_CASE__=[1, 2, 3, 6]		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=0.4		,SCREAMING_SNAKE_CASE__=2_56		,SCREAMING_SNAKE_CASE__=1		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=2_55		,**SCREAMING_SNAKE_CASE__		,)			->      Optional[Any]:





             """simple docstring"""






             super().__init__(**SCREAMING_SNAKE_CASE__							)

             __SCREAMING_SNAKE_CASE   :Any						=  vocab_size
             __SCREAMING_SNAKE_CASE   :Dict						=  hidden_size
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  num_hidden_layers
             __SCREAMING_SNAKE_CASE   :List[str]						=  num_attention_heads
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  intermediate_size
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  hidden_act
             __SCREAMING_SNAKE_CASE   :Tuple						=  hidden_dropout_prob
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  attention_probs_dropout_prob
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  initializer_range
             __SCREAMING_SNAKE_CASE   :str						=  layer_norm_eps

             __SCREAMING_SNAKE_CASE   :int						=  image_size
             __SCREAMING_SNAKE_CASE   :Tuple						=  patch_size
             __SCREAMING_SNAKE_CASE   :Any						=  num_channels
             __SCREAMING_SNAKE_CASE   :Any						=  use_mask_token
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  use_absolute_position_embeddings
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  use_relative_position_bias
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  use_shared_relative_position_bias
             __SCREAMING_SNAKE_CASE   :List[str]						=  layer_scale_init_value
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  drop_path_rate
             __SCREAMING_SNAKE_CASE   :str						=  use_mean_pooling
             # decode head attributes (semantic segmentation)
             __SCREAMING_SNAKE_CASE   :Dict						=  out_indices
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  pool_scales
             # auxiliary head attributes (semantic segmentation)
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  use_auxiliary_head
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  auxiliary_loss_weight
             __SCREAMING_SNAKE_CASE   :Dict						=  auxiliary_channels
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  auxiliary_num_convs
             __SCREAMING_SNAKE_CASE   :List[str]						=  auxiliary_concat_input
             __SCREAMING_SNAKE_CASE   :List[Any]						=  semantic_loss_ignore_index






class 					_SCREAMING_SNAKE_CASE(   A  ):
       SCREAMING_SNAKE_CASE_   :      List[Any]							=  version.parse('''1.11'''  )




       @property
       def     _UpperCamelCase     (						self							)			->      Mapping[str, Mapping[int, str]]:





             """simple docstring"""






             return OrderedDict(
                 [
                     ('''pixel_values''', {0: '''batch''', 1: '''num_channels''', 2: '''height''', 3: '''width'''}),
                 ]							)




       @property
       def     _UpperCamelCase     (						self							)			->      float:





             """simple docstring"""






             return 1E-4