
"""simple docstring"""






import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


__UpperCAmelCase 					=    logging.get_logger(__name__)

__UpperCAmelCase 					=    {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

__UpperCAmelCase 					=    {
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

__UpperCAmelCase 					=    {
    'openai-gpt': 5_12,
}






def 		_snake_case			(      lowercase__						:    Optional[Any]						)    ->							int:


			'''simple docstring'''





			lowerCAmelCase_       :Optional[Any]     	=    text.strip()
			if not text:
						return []
			lowerCAmelCase_       :List[str]     	=    text.split()
			return tokens




class      _SCREAMING_SNAKE_CASE   (     __SCREAMING_SNAKE_CASE				):






			def __init__(   self			,  __A=True			,  __A=None			,  __A=True			,  __A=None			,  __A=True			,  )     ->	Tuple:
						if never_split is None:
									lowerCAmelCase_       :str     	=    []
						lowerCAmelCase_       :Tuple     	=    do_lower_case
						lowerCAmelCase_       :Optional[Any]     	=    set(_A			)
						lowerCAmelCase_       :Optional[Any]     	=    tokenize_chinese_chars
						lowerCAmelCase_       :Tuple     	=    strip_accents
						lowerCAmelCase_       :Any     	=    do_split_on_punc






			def 					__lowerCAmelCase					(   self			,  __A			,  __A=None			)     ->	Dict:
						lowerCAmelCase_       :str     	=    self.never_split.union(set(_A			)			) if never_split else self.never_split
						lowerCAmelCase_       :Optional[Any]     	=    self._clean_text(_A			)

						# This was added on November 1st, 2018 for the multilingual and Chinese
						# models. This is also applied to the English models now, but it doesn't
						# matter since the English models were not trained on any Chinese data
						# and generally don't have any Chinese data in them (there are Chinese
						# characters in the vocabulary because Wikipedia does have some Chinese
						# words in the English Wikipedia.).
						if self.tokenize_chinese_chars:
									lowerCAmelCase_       :Optional[Any]     	=    self._tokenize_chinese_chars(_A			)
						# prevents treating the same character with different unicode codepoints as different characters
						lowerCAmelCase_       :Any     	=    unicodedata.normalize("""NFC"""			,  _A			)
						lowerCAmelCase_       :Union[str, Any]     	=    whitespace_tokenize(_A			)
						lowerCAmelCase_       :List[str]     	=    []
						for token in orig_tokens:
									if token not in never_split:
												if self.do_lower_case:
															lowerCAmelCase_       :Dict     	=    token.lower()
															if self.strip_accents is not False:
																		lowerCAmelCase_       :Optional[Any]     	=    self._run_strip_accents(_A			)
												elif self.strip_accents:
															lowerCAmelCase_       :int     	=    self._run_strip_accents(_A			)
									split_tokens.extend(self._run_split_on_punc(_A			,  _A			)			)

						lowerCAmelCase_       :List[Any]     	=    whitespace_tokenize(""" """.join(_A			)			)
						return output_tokens






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Any:
						lowerCAmelCase_       :List[str]     	=    unicodedata.normalize("""NFD"""			,  _A			)
						lowerCAmelCase_       :Tuple     	=    []
						for char in text:
									lowerCAmelCase_       :Any     	=    unicodedata.category(_A			)
									if cat == "Mn":
												continue
									output.append(_A			)
						return "".join(_A			)






			def 					__lowerCAmelCase					(   self			,  __A			,  __A=None			)     ->	Optional[int]:
						if not self.do_split_on_punc or (never_split is not None and text in never_split):
									return [text]
						lowerCAmelCase_       :List[Any]     	=    list(_A			)
						lowerCAmelCase_       :Optional[Any]     	=    0
						lowerCAmelCase_       :Dict     	=    True
						lowerCAmelCase_       :Optional[int]     	=    []
						while i < len(_A			):
									lowerCAmelCase_       :Dict     	=    chars[i]
									if _is_punctuation(_A			):
												output.append([char]			)
												lowerCAmelCase_       :str     	=    True
									else:
												if start_new_word:
															output.append([]			)
												lowerCAmelCase_       :Optional[Any]     	=    False
												output[-1].append(_A			)
									i += 1

						return ["".join(_A			) for x in output]






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	str:
						lowerCAmelCase_       :Any     	=    []
						for char in text:
									lowerCAmelCase_       :List[Any]     	=    ord(_A			)
									if self._is_chinese_char(_A			):
												output.append(""" """			)
												output.append(_A			)
												output.append(""" """			)
									else:
												output.append(_A			)
						return "".join(_A			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	str:
						# This defines a "chinese character" as anything in the CJK Unicode block:
						#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
						#
						# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
						# despite its name. The modern Korean Hangul alphabet is a different block,
						# as is Japanese Hiragana and Katakana. Those alphabets are used to write
						# space-separated words, so they are not treated specially and handled
						# like the all of the other languages.
						if (
						    (cp >= 0x4E_00 and cp <= 0x9F_FF)
						    or (cp >= 0x34_00 and cp <= 0x4D_BF)  #
						    or (cp >= 0x2_00_00 and cp <= 0x2_A6_DF)  #
						    or (cp >= 0x2_A7_00 and cp <= 0x2_B7_3F)  #
						    or (cp >= 0x2_B7_40 and cp <= 0x2_B8_1F)  #
						    or (cp >= 0x2_B8_20 and cp <= 0x2_CE_AF)  #
						    or (cp >= 0xF9_00 and cp <= 0xFA_FF)
						    or (cp >= 0x2_F8_00 and cp <= 0x2_FA_1F)  #
						):  #
									return True

						return False






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Optional[Any]:
						lowerCAmelCase_       :Union[str, Any]     	=    []
						for char in text:
									lowerCAmelCase_       :Optional[Any]     	=    ord(_A			)
									if cp == 0 or cp == 0xFF_FD or _is_control(_A			):
												continue
									if _is_whitespace(_A			):
												output.append(""" """			)
									else:
												output.append(_A			)
						return "".join(_A			)






def 		_snake_case			(      lowercase__						:    Union[str, Any]						)    ->							int:


			'''simple docstring'''





			lowerCAmelCase_       :Tuple     	=    set()
			lowerCAmelCase_       :int     	=    word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)						)
						lowerCAmelCase_       :List[str]     	=    char
			return pairs






def 		_snake_case			(      lowercase__						:    List[str]						)    ->							str:


			'''simple docstring'''





			lowerCAmelCase_       :Union[str, Any]     	=    text.replace("""—""" ,	"""-"""						)
			lowerCAmelCase_       :str     	=    text.replace("""–""" ,	"""-"""						)
			lowerCAmelCase_       :Optional[Any]     	=    text.replace("""―""" ,	"""-"""						)
			lowerCAmelCase_       :Any     	=    text.replace("""…""" ,	"""..."""						)
			lowerCAmelCase_       :Tuple     	=    text.replace("""´""" ,	"""\'"""						)
			lowerCAmelCase_       :Optional[Any]     	=    re.sub(r"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)""" ,	r""" \1 """ ,	__lowerCamelCase						)
			lowerCAmelCase_       :Tuple     	=    re.sub(r"""\s*\n\s*""" ,	""" \n """ ,	__lowerCamelCase						)
			lowerCAmelCase_       :Tuple     	=    re.sub(r"""[^\S\n]+""" ,	""" """ ,	__lowerCamelCase						)
			return text.strip()




class      _SCREAMING_SNAKE_CASE   (     __SCREAMING_SNAKE_CASE				):
			UpperCAmelCase_    :List[Any]												=		VOCAB_FILES_NAMES
			UpperCAmelCase_    :Any												=		PRETRAINED_VOCAB_FILES_MAP
			UpperCAmelCase_    :Tuple												=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCAmelCase_    :str												=		["input_ids", "attention_mask"]






			def __init__(   self			,  __A			,  __A			,  __A="<unk>"			,  **__A			)     ->	Optional[Any]:
						super().__init__(unk_token=_A			,  **_A			)

						try:
									import ftfy
									from spacy.lang.en import English

									lowerCAmelCase_       :Optional[Any]     	=    English()
									lowerCAmelCase_       :List[Any]     	=    _nlp.tokenizer
									lowerCAmelCase_       :Union[str, Any]     	=    ftfy.fix_text
						except ImportError:
									logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""			)
									lowerCAmelCase_       :List[str]     	=    BasicTokenizer(do_lower_case=_A			)
									lowerCAmelCase_       :Dict     	=    None

						with open(_A			,  encoding="""utf-8"""			) as vocab_handle:
									lowerCAmelCase_       :Any     	=    json.load(_A			)
						lowerCAmelCase_       :int     	=    {v: k for k, v in self.encoder.items()}
						with open(_A			,  encoding="""utf-8"""			) as merges_handle:
									lowerCAmelCase_       :List[Any]     	=    merges_handle.read().split("""\n"""			)[1:-1]
						lowerCAmelCase_       :str     	=    [tuple(merge.split()			) for merge in merges]
						lowerCAmelCase_       :Optional[Any]     	=    dict(zip(_A			,  range(len(_A			)			)			)			)
						lowerCAmelCase_       :int     	=    {}






			@property
			def 					__lowerCAmelCase					(   self			)     ->	Dict:
						return True






			@property
			def 					__lowerCAmelCase					(   self			)     ->	List[str]:
						return len(self.encoder			)






			def 					__lowerCAmelCase					(   self			)     ->	Optional[int]:
						return dict(self.encoder			,  **self.added_tokens_encoder			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	List[str]:
						lowerCAmelCase_       :int     	=    tuple(token[:-1]			) + (token[-1] + """</w>""",)
						if token in self.cache:
									return self.cache[token]
						lowerCAmelCase_       :int     	=    get_pairs(_A			)

						if not pairs:
									return token + "</w>"

						while True:
									lowerCAmelCase_       :Optional[int]     	=    min(_A			,  key=lambda __A			: self.bpe_ranks.get(_A			,  float("""inf"""			)			)			)
									if bigram not in self.bpe_ranks:
												break
									lowerCAmelCase_	,			lowerCAmelCase_       :Union[str, Any]     	=    bigram
									lowerCAmelCase_       :Optional[int]     	=    []
									lowerCAmelCase_       :Tuple     	=    0
									while i < len(_A			):
												try:
															lowerCAmelCase_       :List[str]     	=    word.index(_A			,  _A			)
												except ValueError:
															new_word.extend(word[i:]			)
															break
												else:
															new_word.extend(word[i:j]			)
															lowerCAmelCase_       :int     	=    j

												if word[i] == first and i < len(_A			) - 1 and word[i + 1] == second:
															new_word.append(first + second			)
															i += 2
												else:
															new_word.append(word[i]			)
															i += 1
									lowerCAmelCase_       :Union[str, Any]     	=    tuple(_A			)
									lowerCAmelCase_       :Optional[Any]     	=    new_word
									if len(_A			) == 1:
												break
									else:
												lowerCAmelCase_       :Dict     	=    get_pairs(_A			)
						lowerCAmelCase_       :Union[str, Any]     	=    """ """.join(_A			)
						if word == "\n  </w>":
									lowerCAmelCase_       :Optional[int]     	=    """\n</w>"""
						lowerCAmelCase_       :Any     	=    word
						return word






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	str:
						lowerCAmelCase_       :int     	=    []
						if self.fix_text is None:
									# Using BERT's BasicTokenizer
									lowerCAmelCase_       :Optional[Any]     	=    self.nlp.tokenize(_A			)
									for token in text:
												split_tokens.extend(list(self.bpe(_A			).split(""" """			)			)			)
						else:
									# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
									lowerCAmelCase_       :Optional[Any]     	=    self.nlp(text_standardize(self.fix_text(_A			)			)			)
									for token in text:
												split_tokens.extend(list(self.bpe(token.text.lower()			).split(""" """			)			)			)
						return split_tokens






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	Optional[int]:
						return self.encoder.get(_A			,  self.encoder.get(self.unk_token			)			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	List[str]:
						return self.decoder.get(_A			,  self.unk_token			)






			def 					__lowerCAmelCase					(   self			,  __A			)     ->	str:
						lowerCAmelCase_       :List[str]     	=    """""".join(_A			).replace("""</w>"""			,  """ """			).strip()
						return out_string






			def 					__lowerCAmelCase					(   self			,  __A			,  __A = None			)     ->	Tuple[str]:
						if not os.path.isdir(_A			):
									logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""			)
									return
						lowerCAmelCase_       :Dict     	=    os.path.join(
						    _A			,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)
						lowerCAmelCase_       :Dict     	=    os.path.join(
						    _A			,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]			)

						with open(_A			,  """w"""			,  encoding="""utf-8"""			) as f:
									f.write(json.dumps(self.encoder			,  indent=2			,  sort_keys=_A			,  ensure_ascii=_A			) + """\n"""			)

						lowerCAmelCase_       :Any     	=    0
						with open(_A			,  """w"""			,  encoding="""utf-8"""			) as writer:
									writer.write("""#version: 0.2\n"""			)
									for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,  key=lambda __A			: kv[1]			):
												if index != token_index:
															logger.warning(
															    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
															    """ Please check that the tokenizer is not corrupted!"""			)
															lowerCAmelCase_       :List[Any]     	=    token_index
												writer.write(""" """.join(_A			) + """\n"""			)
												index += 1

						return vocab_file, merge_file








from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


if TYPE_CHECKING:
   from ... import FeatureExtractionMixin, PreTrainedTokenizerBase, TensorType


__UpperCAmelCase   						=     logging.get_logger(__name__)

__UpperCAmelCase   						=     {
    "microsoft/deberta-base": "https://huggingface.co/microsoft/deberta-base/resolve/main/config.json",
    "microsoft/deberta-large": "https://huggingface.co/microsoft/deberta-large/resolve/main/config.json",
    "microsoft/deberta-xlarge": "https://huggingface.co/microsoft/deberta-xlarge/resolve/main/config.json",
    "microsoft/deberta-base-mnli": "https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/config.json",
    "microsoft/deberta-large-mnli": "https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/config.json",
    "microsoft/deberta-xlarge-mnli": "https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/config.json",
}





class 				UpperCamelCase__		(	__SCREAMING_SNAKE_CASE	):






       """simple docstring"""


       UpperCAmelCase_				       ="deberta"



       def __init__(							self ,     _A=50265 ,     _A=768 ,     _A=12 ,     _A=12 ,     _A=3072 ,     _A="gelu" ,     _A=0.1 ,     _A=0.1 ,     _A=512 ,     _A=0 ,     _A=0.02 ,     _A=1E-7 ,     _A=False ,     _A=-1 ,     _A=0 ,     _A=True ,     _A=None ,     _A=0 ,     _A="gelu" ,     **_A ,     )				->				Optional[int]:
          super().__init__(**_A	)

          SCREAMING_SNAKE_CASE_	=      hidden_size
          SCREAMING_SNAKE_CASE_	=      num_hidden_layers
          SCREAMING_SNAKE_CASE_	=      num_attention_heads
          SCREAMING_SNAKE_CASE_	=      intermediate_size
          SCREAMING_SNAKE_CASE_	=      hidden_act
          SCREAMING_SNAKE_CASE_	=      hidden_dropout_prob
          SCREAMING_SNAKE_CASE_	=      attention_probs_dropout_prob
          SCREAMING_SNAKE_CASE_	=      max_position_embeddings
          SCREAMING_SNAKE_CASE_	=      type_vocab_size
          SCREAMING_SNAKE_CASE_	=      initializer_range
          SCREAMING_SNAKE_CASE_	=      relative_attention
          SCREAMING_SNAKE_CASE_	=      max_relative_positions
          SCREAMING_SNAKE_CASE_	=      pad_token_id
          SCREAMING_SNAKE_CASE_	=      position_biased_input

          # Backwards compatibility
          if type(_A	) == str:
             SCREAMING_SNAKE_CASE_	=      [x.strip() for x in pos_att_type.lower().split('''|'''	)]

          SCREAMING_SNAKE_CASE_	=      pos_att_type
          SCREAMING_SNAKE_CASE_	=      vocab_size
          SCREAMING_SNAKE_CASE_	=      layer_norm_eps

          SCREAMING_SNAKE_CASE_	=      kwargs.get('''pooler_hidden_size''' ,     _A	)
          SCREAMING_SNAKE_CASE_	=      pooler_dropout
          SCREAMING_SNAKE_CASE_	=      pooler_hidden_act







class 				UpperCamelCase__		(	__SCREAMING_SNAKE_CASE	):






       """simple docstring"""


       @property
       def     _UpperCamelCase	(							self	)				->				Mapping[str, Mapping[int, str]]:
          if self.task == "multiple-choice":
             SCREAMING_SNAKE_CASE_	=      {0: '''batch''', 1: '''choice''', 2: '''sequence'''}
          else:
             SCREAMING_SNAKE_CASE_	=      {0: '''batch''', 1: '''sequence'''}
          if self._config.type_vocab_size > 0:
             return OrderedDict(
                 [('''input_ids''', dynamic_axis), ('''attention_mask''', dynamic_axis), ('''token_type_ids''', dynamic_axis)]	)
          else:
             return OrderedDict([('''input_ids''', dynamic_axis), ('''attention_mask''', dynamic_axis)]	)



       @property
       def     _UpperCamelCase	(							self	)				->				int:
          return 12




       def     _UpperCamelCase	(							self ,     _A ,     _A = -1 ,     _A = -1 ,     _A = -1 ,     _A = False ,     _A = None ,     _A = 3 ,     _A = 40 ,     _A = 40 ,     _A = None ,     )				->				Mapping[str, Any]:
          SCREAMING_SNAKE_CASE_	=      super().generate_dummy_inputs(preprocessor=_A ,     framework=_A	)
          if self._config.type_vocab_size == 0 and "token_type_ids" in dummy_inputs:
             del dummy_inputs["token_type_ids"]
          return dummy_inputs


