




'''simple docstring'''




import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation

snake_case__ 							=    logging.get_logger(__name__)

snake_case__ 							=    {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

snake_case__ 							=    {
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

snake_case__ 							=    {
    """microsoft/deberta-base""": 5_12,
    """microsoft/deberta-large""": 5_12,
    """microsoft/deberta-xlarge""": 5_12,
    """microsoft/deberta-base-mnli""": 5_12,
    """microsoft/deberta-large-mnli""": 5_12,
    """microsoft/deberta-xlarge-mnli""": 5_12,
}

snake_case__ 							=    {
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}




def     snake_case__     (			) -> Optional[int]:
					A_			:  Tuple			=				(
					    list(range(ord('''!'''      )      ,		ord('''~'''      ) + 1      )      ) + list(range(ord('''¡'''      )      ,		ord('''¬'''      ) + 1      )      ) + list(range(ord('''®'''      )      ,		ord('''ÿ'''      ) + 1      )      )
					)
					A_			:  Optional[int]			=				bs[:]
					A_			:  List[str]			=				0
					for b in range(2**8      ):
										if b not in bs:
															bs.append(lowerCamelCase__      )
															cs.append(2**8 + n      )
															n += 1
					A_			:  int			=				[chr(lowerCamelCase__      ) for n in cs]
					return dict(zip(lowerCamelCase__      ,		lowerCamelCase__      )      )




def     snake_case__     (			lowerCamelCase__					: Dict      ) -> List[str]:
					A_			:  List[str]			=				set()
					A_			:  Any			=				word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)      )
										A_			:  Any			=				char
					return pairs






class      UpperCamelCase_  (a__  ):







						"""simple docstring"""


						_lowerCAmelCase         =       VOCAB_FILES_NAMES
						_lowerCAmelCase         =       PRETRAINED_VOCAB_FILES_MAP
						_lowerCAmelCase         =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_lowerCAmelCase         =       ['input_ids', 'attention_mask', 'token_type_ids']







						def __init__(  self   :							int				,			_lowerCamelCase   :							Union[str, Any]				,			_lowerCamelCase   :							Union[str, Any]				,			_lowerCamelCase   :							List[Any]="replace"				,			_lowerCamelCase   :							Optional[int]="[CLS]"				,			_lowerCamelCase   :							str="[SEP]"				,			_lowerCamelCase   :							Optional[Any]="[SEP]"				,			_lowerCamelCase   :							int="[CLS]"				,			_lowerCamelCase   :							str="[UNK]"				,			_lowerCamelCase   :							Tuple="[PAD]"				,			_lowerCamelCase   :							Union[str, Any]="[MASK]"				,			_lowerCamelCase   :							Union[str, Any]=False				,			_lowerCamelCase   :							str=False				,			**_lowerCamelCase   :							Optional[int]				,			):





											"""simple docstring"""







											A_			:  List[Any]			=				AddedToken(_lowerCamelCase				,			lstrip=_lowerCamelCase				,			rstrip=_lowerCamelCase     ) if isinstance(_lowerCamelCase				,			_lowerCamelCase     ) else bos_token
											A_			:  int			=				AddedToken(_lowerCamelCase				,			lstrip=_lowerCamelCase				,			rstrip=_lowerCamelCase     ) if isinstance(_lowerCamelCase				,			_lowerCamelCase     ) else eos_token
											A_			:  Any			=				AddedToken(_lowerCamelCase				,			lstrip=_lowerCamelCase				,			rstrip=_lowerCamelCase     ) if isinstance(_lowerCamelCase				,			_lowerCamelCase     ) else sep_token
											A_			:  Optional[Any]			=				AddedToken(_lowerCamelCase				,			lstrip=_lowerCamelCase				,			rstrip=_lowerCamelCase     ) if isinstance(_lowerCamelCase				,			_lowerCamelCase     ) else cls_token
											A_			:  str			=				AddedToken(_lowerCamelCase				,			lstrip=_lowerCamelCase				,			rstrip=_lowerCamelCase     ) if isinstance(_lowerCamelCase				,			_lowerCamelCase     ) else unk_token
											A_			:  Optional[int]			=				AddedToken(_lowerCamelCase				,			lstrip=_lowerCamelCase				,			rstrip=_lowerCamelCase     ) if isinstance(_lowerCamelCase				,			_lowerCamelCase     ) else pad_token

											# Mask token behave like a normal word, i.e. include the space before it
											A_			:  Dict			=				AddedToken(_lowerCamelCase				,			lstrip=_lowerCamelCase				,			rstrip=_lowerCamelCase     ) if isinstance(_lowerCamelCase				,			_lowerCamelCase     ) else mask_token

											super().__init__(
											    errors=_lowerCamelCase				,			bos_token=_lowerCamelCase				,			eos_token=_lowerCamelCase				,			unk_token=_lowerCamelCase				,			sep_token=_lowerCamelCase				,			cls_token=_lowerCamelCase				,			pad_token=_lowerCamelCase				,			mask_token=_lowerCamelCase				,			add_prefix_space=_lowerCamelCase				,			add_bos_token=_lowerCamelCase				,			**_lowerCamelCase				,			)
											A_			:  Tuple			=				add_bos_token

											with open(_lowerCamelCase				,			encoding='''utf-8'''     ) as vocab_handle:
																A_			:  int			=				json.load(_lowerCamelCase     )
											A_			:  List[str]			=				{v: k for k, v in self.encoder.items()}
											A_			:  Any			=				errors  # how to handle errors in decoding
											A_			:  List[str]			=				bytes_to_unicode()
											A_			:  Union[str, Any]			=				{v: k for k, v in self.byte_encoder.items()}
											with open(_lowerCamelCase				,			encoding='''utf-8'''     ) as merges_handle:
																A_			:  List[Any]			=				merges_handle.read().split('''\n'''     )[1:-1]
											A_			:  List[Any]			=				[tuple(merge.split()     ) for merge in bpe_merges]
											A_			:  Any			=				dict(zip(_lowerCamelCase				,			range(len(_lowerCamelCase     )     )     )     )
											A_			:  Dict			=				{}
											A_			:  Union[str, Any]			=				add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											A_			:  Tuple			=				re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''     )







						@property
						# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
						def    _a     (  self   :							List[Any]     ):





											"""simple docstring"""







											return len(self.encoder     )







						def    _a     (  self   :							Any     ):





											"""simple docstring"""







											return dict(self.encoder				,			**self.added_tokens_encoder     )







						def    _a     (  self   :							Union[str, Any]				,			_lowerCamelCase   :							List[Any]     ):





											"""simple docstring"""







											if token in self.cache:
																return self.cache[token]
											A_			:  Tuple			=				tuple(_lowerCamelCase     )
											A_			:  Dict			=				get_pairs(_lowerCamelCase     )

											if not pairs:
																return token

											while True:
																A_			:  str			=				min(_lowerCamelCase				,			key=lambda _lowerCamelCase     : self.bpe_ranks.get(_lowerCamelCase				,			float('''inf'''     )     )     )
																if bigram not in self.bpe_ranks:
																					break
																A_			,A_			:  Optional[int]			=				bigram
																A_			:  List[Any]			=				[]
																A_			:  List[Any]			=				0
																while i < len(_lowerCamelCase     ):
																					try:
																										A_			:  str			=				word.index(_lowerCamelCase				,			_lowerCamelCase     )
																					except ValueError:
																										new_word.extend(word[i:]     )
																										break
																					else:
																										new_word.extend(word[i:j]     )
																										A_			:  List[str]			=				j

																					if word[i] == first and i < len(_lowerCamelCase     ) - 1 and word[i + 1] == second:
																										new_word.append(first + second     )
																										i += 2
																					else:
																										new_word.append(word[i]     )
																										i += 1
																A_			:  List[str]			=				tuple(_lowerCamelCase     )
																A_			:  Union[str, Any]			=				new_word
																if len(_lowerCamelCase     ) == 1:
																					break
																else:
																					A_			:  Optional[Any]			=				get_pairs(_lowerCamelCase     )
											A_			:  Dict			=				''' '''.join(_lowerCamelCase     )
											A_			:  int			=				word
											return word







						def    _a     (  self   :							Optional[int]				,			_lowerCamelCase   :							List[int]				,			_lowerCamelCase   :							Optional[List[int]] = None     ):





											"""simple docstring"""







											if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											A_			:  Dict			=				[self.cls_token_id]
											A_			:  Optional[Any]			=				[self.sep_token_id]
											return cls + token_ids_a + sep + token_ids_a + sep







						def    _a     (  self   :							Optional[Any]				,			_lowerCamelCase   :							List[int]				,			_lowerCamelCase   :							Optional[List[int]] = None				,			_lowerCamelCase   :							bool = False     ):





											"""simple docstring"""







											if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_lowerCamelCase				,			token_ids_a=_lowerCamelCase				,			already_has_special_tokens=_lowerCamelCase     )

											if token_ids_a is None:
																return [1] + ([0] * len(_lowerCamelCase     )) + [1]
											return [1] + ([0] * len(_lowerCamelCase     )) + [1] + ([0] * len(_lowerCamelCase     )) + [1]







						def    _a     (  self   :							Dict				,			_lowerCamelCase   :							List[int]				,			_lowerCamelCase   :							Optional[List[int]] = None     ):





											"""simple docstring"""







											A_			:  Optional[Any]			=				[self.sep_token_id]
											A_			:  Dict			=				[self.cls_token_id]

											if token_ids_a is None:
																return len(cls + token_ids_a + sep     ) * [0]
											return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]







						def    _a     (  self   :							str				,			_lowerCamelCase   :							Tuple     ):





											"""simple docstring"""







											A_			:  Optional[Any]			=				[]
											for token in re.findall(self.pat				,			_lowerCamelCase     ):
																A_			:  int			=				''''''.join(
																    self.byte_encoder[b] for b in token.encode('''utf-8'''     )     )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCamelCase     ).split(''' '''     )     )
											return bpe_tokens







						def    _a     (  self   :							int				,			_lowerCamelCase   :							List[Any]     ):





											"""simple docstring"""







											return self.encoder.get(_lowerCamelCase				,			self.encoder.get(self.unk_token     )     )







						def    _a     (  self   :							Tuple				,			_lowerCamelCase   :							Optional[Any]     ):





											"""simple docstring"""







											return self.decoder.get(_lowerCamelCase     )







						def    _a     (  self   :							Optional[Any]				,			_lowerCamelCase   :							Optional[int]     ):





											"""simple docstring"""







											A_			:  int			=				''''''.join(_lowerCamelCase     )
											A_			:  Optional[Any]			=				bytearray([self.byte_decoder[c] for c in text]     ).decode('''utf-8'''				,			errors=self.errors     )
											return text







						def    _a     (  self   :							List[Any]				,			_lowerCamelCase   :							str				,			_lowerCamelCase   :							Optional[str] = None     ):





											"""simple docstring"""







											if not os.path.isdir(_lowerCamelCase     ):
																logger.error(f'Vocabulary path ({save_directory}) should be a directory'     )
																return
											A_			:  Dict			=				os.path.join(
											    _lowerCamelCase				,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )
											A_			:  int			=				os.path.join(
											    _lowerCamelCase				,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']     )

											with open(_lowerCamelCase				,			'''w'''				,			encoding='''utf-8'''     ) as f:
																f.write(json.dumps(self.encoder				,			indent=2				,			sort_keys=_lowerCamelCase				,			ensure_ascii=_lowerCamelCase     ) + '''\n'''     )

											A_			:  Union[str, Any]			=				0
											with open(_lowerCamelCase				,			'''w'''				,			encoding='''utf-8'''     ) as writer:
																writer.write('''#version: 0.2\n'''     )
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,			key=lambda _lowerCamelCase     : kv[1]     ):
																					if index != token_index:
																										logger.warning(
																										    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																										    ''' Please check that the tokenizer is not corrupted!'''     )
																										A_			:  int			=				token_index
																					writer.write(''' '''.join(_lowerCamelCase     ) + '''\n'''     )
																					index += 1

											return vocab_file, merge_file







						def    _a     (  self   :							List[Any]				,			_lowerCamelCase   :							List[str]				,			_lowerCamelCase   :							Any=False				,			**_lowerCamelCase   :							Tuple     ):





											"""simple docstring"""







											A_			:  List[str]			=				kwargs.pop('''add_prefix_space'''				,			self.add_prefix_space     )
											if (is_split_into_words or add_prefix_space) and (len(_lowerCamelCase     ) > 0 and not text[0].isspace()):
																A_			:  Dict			=				''' ''' + text
											return (text, kwargs)







						def    _a     (  self   :							Tuple				,			_lowerCamelCase   :							"Conversation"     ):





											"""simple docstring"""







											A_			:  int			=				[]
											for is_user, text in conversation.iter_texts():
																input_ids.extend(self.encode(_lowerCamelCase				,			add_special_tokens=_lowerCamelCase     ) + [self.eos_token_id]     )
											if len(_lowerCamelCase     ) > self.model_max_length:
																A_			:  Optional[int]			=				input_ids[-self.model_max_length :]
											return input_ids









'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case__ 							=    logging.get_logger(__name__)

snake_case__ 							=    {
    """uw-madison/nystromformer-512""": """https://huggingface.co/uw-madison/nystromformer-512/resolve/main/config.json""",
    # See all Nystromformer models at https://huggingface.co/models?filter=nystromformer
}






class      UpperCamelCase_  (a__  ):







						"""simple docstring"""


						_lowerCAmelCase         =       'nystromformer'







						def __init__(  self   :							Optional[Any]				,			_lowerCamelCase   :							str=30000				,			_lowerCamelCase   :							Optional[Any]=768				,			_lowerCamelCase   :							Optional[Any]=12				,			_lowerCamelCase   :							str=12				,			_lowerCamelCase   :							List[str]=3072				,			_lowerCamelCase   :							Any="gelu_new"				,			_lowerCamelCase   :							List[Any]=0.1				,			_lowerCamelCase   :							List[str]=0.1				,			_lowerCamelCase   :							List[str]=510				,			_lowerCamelCase   :							Optional[Any]=2				,			_lowerCamelCase   :							List[Any]=64				,			_lowerCamelCase   :							str=64				,			_lowerCamelCase   :							str=65				,			_lowerCamelCase   :							List[Any]=False				,			_lowerCamelCase   :							List[Any]=0.02				,			_lowerCamelCase   :							List[Any]=1E-5				,			_lowerCamelCase   :							Dict=1				,			_lowerCamelCase   :							Tuple=0				,			_lowerCamelCase   :							List[str]=2				,			**_lowerCamelCase   :							List[str]				,			):





											"""simple docstring"""







											A_			:  List[Any]			=				vocab_size
											A_			:  Tuple			=				max_position_embeddings
											A_			:  Dict			=				hidden_size
											A_			:  Optional[Any]			=				num_hidden_layers
											A_			:  Dict			=				num_attention_heads
											A_			:  Tuple			=				intermediate_size
											A_			:  Dict			=				hidden_act
											A_			:  List[Any]			=				hidden_dropout_prob
											A_			:  str			=				attention_probs_dropout_prob
											A_			:  Optional[Any]			=				initializer_range
											A_			:  List[str]			=				type_vocab_size
											A_			:  List[Any]			=				segment_means_seq_len
											A_			:  Dict			=				num_landmarks
											A_			:  int			=				conv_kernel_size
											A_			:  Optional[Any]			=				inv_coeff_init_option
											A_			:  str			=				layer_norm_eps
											super().__init__(pad_token_id=_lowerCamelCase				,			bos_token_id=_lowerCamelCase				,			eos_token_id=_lowerCamelCase				,			**_lowerCamelCase     )



