
"""simple docstring"""







import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation

__a			 =   logging.get_logger(__name__)

__a			 =   {"vocab_file": "vocab.json", "merges_file": "merges.txt"}

__a			 =   {
    "vocab_file": {
        "microsoft/deberta-base": "https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json",
        "microsoft/deberta-large": "https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json",
        "microsoft/deberta-xlarge": "https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json",
        "microsoft/deberta-base-mnli": "https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json",
        "microsoft/deberta-large-mnli": "https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json",
        "microsoft/deberta-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "microsoft/deberta-base": "https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt",
        "microsoft/deberta-large": "https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt",
        "microsoft/deberta-xlarge": "https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt",
        "microsoft/deberta-base-mnli": "https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt",
        "microsoft/deberta-large-mnli": "https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt",
        "microsoft/deberta-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"
        ),
    },
}

__a			 =   {
    "microsoft/deberta-base": 5_12,
    "microsoft/deberta-large": 5_12,
    "microsoft/deberta-xlarge": 5_12,
    "microsoft/deberta-base-mnli": 5_12,
    "microsoft/deberta-large-mnli": 5_12,
    "microsoft/deberta-xlarge-mnli": 5_12,
}

__a			 =   {
    "microsoft/deberta-base": {"do_lower_case": False},
    "microsoft/deberta-large": {"do_lower_case": False},
}






def 					a__ (       ):





						"""simple docstring"""

						UpperCAmelCase__		=    (
						    list(range(ord('''!''' )	,			ord('''~''' ) + 1 ) ) + list(range(ord('''¡''' )	,			ord('''¬''' ) + 1 ) ) + list(range(ord('''®''' )	,			ord('''ÿ''' ) + 1 ) )
						)
						UpperCAmelCase__		=    bs[:]
						UpperCAmelCase__		=    0
						for b in range(2**8 ):
												if b not in bs:
																		bs.append(_UpperCamelCase )
																		cs.append(2**8 + n )
																		n += 1
						UpperCAmelCase__		=    [chr(_UpperCamelCase ) for n in cs]
						return dict(zip(_UpperCamelCase	,			_UpperCamelCase ) )






def 					a__ (       _UpperCamelCase ):





						"""simple docstring"""

						UpperCAmelCase__		=    set()
						UpperCAmelCase__		=    word[0]
						for char in word[1:]:
												pairs.add((prev_char, char) )
												UpperCAmelCase__		=    char
						return pairs


class 							lowerCamelCase       (			__SCREAMING_SNAKE_CASE ):


							"""simple docstring"""
							SCREAMING_SNAKE_CASE										=      VOCAB_FILES_NAMES
							SCREAMING_SNAKE_CASE										=      PRETRAINED_VOCAB_FILES_MAP
							SCREAMING_SNAKE_CASE										=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							SCREAMING_SNAKE_CASE										=      ["""input_ids""", """attention_mask""", """token_type_ids"""]






							def __init__(						self :     Optional[Any]		, lowerCamelCase_ :     Any		, lowerCamelCase_ :     List[Any]		, lowerCamelCase_ :     str="replace"		, lowerCamelCase_ :     List[str]="[CLS]"		, lowerCamelCase_ :     List[str]="[SEP]"		, lowerCamelCase_ :     Optional[int]="[SEP]"		, lowerCamelCase_ :     str="[CLS]"		, lowerCamelCase_ :     Optional[Any]="[UNK]"		, lowerCamelCase_ :     int="[PAD]"		, lowerCamelCase_ :     int="[MASK]"		, lowerCamelCase_ :     Optional[Any]=False		, lowerCamelCase_ :     Tuple=False		, **lowerCamelCase_ :     Union[str, Any]		, ):
													UpperCAmelCase__		=    AddedToken(lowerCamelCase_		, lstrip=lowerCamelCase_		, rstrip=lowerCamelCase_				) if isinstance(lowerCamelCase_		, lowerCamelCase_				) else bos_token
													UpperCAmelCase__		=    AddedToken(lowerCamelCase_		, lstrip=lowerCamelCase_		, rstrip=lowerCamelCase_				) if isinstance(lowerCamelCase_		, lowerCamelCase_				) else eos_token
													UpperCAmelCase__		=    AddedToken(lowerCamelCase_		, lstrip=lowerCamelCase_		, rstrip=lowerCamelCase_				) if isinstance(lowerCamelCase_		, lowerCamelCase_				) else sep_token
													UpperCAmelCase__		=    AddedToken(lowerCamelCase_		, lstrip=lowerCamelCase_		, rstrip=lowerCamelCase_				) if isinstance(lowerCamelCase_		, lowerCamelCase_				) else cls_token
													UpperCAmelCase__		=    AddedToken(lowerCamelCase_		, lstrip=lowerCamelCase_		, rstrip=lowerCamelCase_				) if isinstance(lowerCamelCase_		, lowerCamelCase_				) else unk_token
													UpperCAmelCase__		=    AddedToken(lowerCamelCase_		, lstrip=lowerCamelCase_		, rstrip=lowerCamelCase_				) if isinstance(lowerCamelCase_		, lowerCamelCase_				) else pad_token

													# Mask token behave like a normal word, i.e. include the space before it
													UpperCAmelCase__		=    AddedToken(lowerCamelCase_		, lstrip=lowerCamelCase_		, rstrip=lowerCamelCase_				) if isinstance(lowerCamelCase_		, lowerCamelCase_				) else mask_token

													super().__init__(
													    errors=lowerCamelCase_		, bos_token=lowerCamelCase_		, eos_token=lowerCamelCase_		, unk_token=lowerCamelCase_		, sep_token=lowerCamelCase_		, cls_token=lowerCamelCase_		, pad_token=lowerCamelCase_		, mask_token=lowerCamelCase_		, add_prefix_space=lowerCamelCase_		, add_bos_token=lowerCamelCase_		, **lowerCamelCase_		, )
													UpperCAmelCase__		=    add_bos_token

													with open(lowerCamelCase_		, encoding='''utf-8'''				) as vocab_handle:
																			UpperCAmelCase__		=    json.load(lowerCamelCase_				)
													UpperCAmelCase__		=    {v: k for k, v in self.encoder.items()}
													UpperCAmelCase__		=    errors  # how to handle errors in decoding
													UpperCAmelCase__		=    bytes_to_unicode()
													UpperCAmelCase__		=    {v: k for k, v in self.byte_encoder.items()}
													with open(lowerCamelCase_		, encoding='''utf-8'''				) as merges_handle:
																			UpperCAmelCase__		=    merges_handle.read().split('''\n'''				)[1:-1]
													UpperCAmelCase__		=    [tuple(merge.split()				) for merge in bpe_merges]
													UpperCAmelCase__		=    dict(zip(lowerCamelCase_		, range(len(lowerCamelCase_				)				)				)				)
													UpperCAmelCase__		=    {}
													UpperCAmelCase__		=    add_prefix_space

													# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
													UpperCAmelCase__		=    re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''				)






							@property
							# Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
							def 					lowerCAmelCase     (						self :     Optional[int]				):
													return len(self.encoder				)






							def 					lowerCAmelCase     (						self :     str				):
													return dict(self.encoder		, **self.added_tokens_encoder				)






							def 					lowerCAmelCase     (						self :     Optional[int]		, lowerCamelCase_ :     Any				):
													if token in self.cache:
																			return self.cache[token]
													UpperCAmelCase__		=    tuple(lowerCamelCase_				)
													UpperCAmelCase__		=    get_pairs(lowerCamelCase_				)

													if not pairs:
																			return token

													while True:
																			UpperCAmelCase__		=    min(lowerCamelCase_		, key=lambda lowerCamelCase_				: self.bpe_ranks.get(lowerCamelCase_		, float('''inf'''				)				)				)
																			if bigram not in self.bpe_ranks:
																									break
																			UpperCAmelCase__	,				UpperCAmelCase__		=    bigram
																			UpperCAmelCase__		=    []
																			UpperCAmelCase__		=    0
																			while i < len(lowerCamelCase_				):
																									try:
																															UpperCAmelCase__		=    word.index(lowerCamelCase_		, lowerCamelCase_				)
																									except ValueError:
																															new_word.extend(word[i:]				)
																															break
																									else:
																															new_word.extend(word[i:j]				)
																															UpperCAmelCase__		=    j

																									if word[i] == first and i < len(lowerCamelCase_				) - 1 and word[i + 1] == second:
																															new_word.append(first + second				)
																															i += 2
																									else:
																															new_word.append(word[i]				)
																															i += 1
																			UpperCAmelCase__		=    tuple(lowerCamelCase_				)
																			UpperCAmelCase__		=    new_word
																			if len(lowerCamelCase_				) == 1:
																									break
																			else:
																									UpperCAmelCase__		=    get_pairs(lowerCamelCase_				)
													UpperCAmelCase__		=    ''' '''.join(lowerCamelCase_				)
													UpperCAmelCase__		=    word
													return word






							def 					lowerCAmelCase     (						self :     List[str]		, lowerCamelCase_ :     List[int]		, lowerCamelCase_ :     Optional[List[int]] = None				):
													if token_ids_a is None:
																			return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
													UpperCAmelCase__		=    [self.cls_token_id]
													UpperCAmelCase__		=    [self.sep_token_id]
													return cls + token_ids_a + sep + token_ids_a + sep






							def 					lowerCAmelCase     (						self :     Dict		, lowerCamelCase_ :     List[int]		, lowerCamelCase_ :     Optional[List[int]] = None		, lowerCamelCase_ :     bool = False				):
													if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=lowerCamelCase_		, token_ids_a=lowerCamelCase_		, already_has_special_tokens=lowerCamelCase_				)

													if token_ids_a is None:
																			return [1] + ([0] * len(lowerCamelCase_				)) + [1]
													return [1] + ([0] * len(lowerCamelCase_				)) + [1] + ([0] * len(lowerCamelCase_				)) + [1]






							def 					lowerCAmelCase     (						self :     List[str]		, lowerCamelCase_ :     List[int]		, lowerCamelCase_ :     Optional[List[int]] = None				):
													UpperCAmelCase__		=    [self.sep_token_id]
													UpperCAmelCase__		=    [self.cls_token_id]

													if token_ids_a is None:
																			return len(cls + token_ids_a + sep				) * [0]
													return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]






							def 					lowerCAmelCase     (						self :     Optional[int]		, lowerCamelCase_ :     Tuple				):
													UpperCAmelCase__		=    []
													for token in re.findall(self.pat		, lowerCamelCase_				):
																			UpperCAmelCase__		=    ''''''.join(
																			    self.byte_encoder[b] for b in token.encode('''utf-8'''				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																			bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCamelCase_				).split(''' '''				)				)
													return bpe_tokens






							def 					lowerCAmelCase     (						self :     Union[str, Any]		, lowerCamelCase_ :     int				):
													return self.encoder.get(lowerCamelCase_		, self.encoder.get(self.unk_token				)				)






							def 					lowerCAmelCase     (						self :     Optional[Any]		, lowerCamelCase_ :     str				):
													return self.decoder.get(lowerCamelCase_				)






							def 					lowerCAmelCase     (						self :     List[Any]		, lowerCamelCase_ :     Union[str, Any]				):
													UpperCAmelCase__		=    ''''''.join(lowerCamelCase_				)
													UpperCAmelCase__		=    bytearray([self.byte_decoder[c] for c in text]				).decode('''utf-8'''		, errors=self.errors				)
													return text






							def 					lowerCAmelCase     (						self :     int		, lowerCamelCase_ :     str		, lowerCamelCase_ :     Optional[str] = None				):
													if not os.path.isdir(lowerCamelCase_				):
																			logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
																			return
													UpperCAmelCase__		=    os.path.join(
													    lowerCamelCase_		, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
													UpperCAmelCase__		=    os.path.join(
													    lowerCamelCase_		, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']				)

													with open(lowerCamelCase_		, '''w'''		, encoding='''utf-8'''				) as f:
																			f.write(json.dumps(self.encoder		, indent=2		, sort_keys=lowerCamelCase_		, ensure_ascii=lowerCamelCase_				) + '''\n'''				)

													UpperCAmelCase__		=    0
													with open(lowerCamelCase_		, '''w'''		, encoding='''utf-8'''				) as writer:
																			writer.write('''#version: 0.2\n'''				)
																			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		, key=lambda lowerCamelCase_				: kv[1]				):
																									if index != token_index:
																															logger.warning(
																															    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																															    ''' Please check that the tokenizer is not corrupted!'''				)
																															UpperCAmelCase__		=    token_index
																									writer.write(''' '''.join(lowerCamelCase_				) + '''\n'''				)
																									index += 1

													return vocab_file, merge_file






							def 					lowerCAmelCase     (						self :     Any		, lowerCamelCase_ :     Optional[int]		, lowerCamelCase_ :     Union[str, Any]=False		, **lowerCamelCase_ :     List[str]				):
													UpperCAmelCase__		=    kwargs.pop('''add_prefix_space'''		, self.add_prefix_space				)
													if (is_split_into_words or add_prefix_space) and (len(lowerCamelCase_				) > 0 and not text[0].isspace()):
																			UpperCAmelCase__		=    ''' ''' + text
													return (text, kwargs)






							def 					lowerCAmelCase     (						self :     List[str]		, lowerCamelCase_ :     "Conversation"				):
													UpperCAmelCase__		=    []
													for is_user, text in conversation.iter_texts():
																			input_ids.extend(self.encode(lowerCamelCase_		, add_special_tokens=lowerCamelCase_				) + [self.eos_token_id]				)
													if len(lowerCamelCase_				) > self.model_max_length:
																			UpperCAmelCase__		=    input_ids[-self.model_max_length :]
													return input_ids



"""simple docstring"""







from __future__ import annotations

from collections.abc import Callable
from typing import Generic, TypeVar

__a			 =   TypeVar("T")
__a			 =   TypeVar("U")
class 							lowerCamelCase       (			Generic[T, U] ):


							"""simple docstring"""






							def __init__(						self :     str		, lowerCamelCase_ :     T | None		, lowerCamelCase_ :     U | None				):
													UpperCAmelCase__		=    key
													UpperCAmelCase__		=    val
													UpperCAmelCase__		=    0
													UpperCAmelCase__		=    None
													UpperCAmelCase__		=    None






							def __repr__(						self :     Tuple				):
													return "Node: key: {}, val: {}, freq: {}, has next: {}, has prev: {}".format(
													    self.key		, self.val		, self.freq		, self.next is not None		, self.prev is not None				)




class 							lowerCamelCase       (			Generic[T, U] ):


							"""simple docstring"""






							def __init__(						self :     List[str]				):
													UpperCAmelCase__		=    DoubleLinkedListNode(lowerCamelCase_		, lowerCamelCase_				)
													UpperCAmelCase__		=    DoubleLinkedListNode(lowerCamelCase_		, lowerCamelCase_				)
													UpperCAmelCase__	,				UpperCAmelCase__		=    self.rear, self.head






							def __repr__(						self :     int				):
													UpperCAmelCase__		=    ['''DoubleLinkedList''']
													UpperCAmelCase__		=    self.head
													while node.next is not None:
																			rep.append(str(lowerCamelCase_				)				)
																			UpperCAmelCase__		=    node.next
													rep.append(str(self.rear				)				)
													return ",\n    ".join(lowerCamelCase_				)






							def 					lowerCAmelCase     (						self :     List[Any]		, lowerCamelCase_ :     DoubleLinkedListNode[T, U]				):
													UpperCAmelCase__		=    self.rear.prev

													# All nodes other than self.head are guaranteed to have non-None previous
													assert previous is not None

													UpperCAmelCase__		=    node
													UpperCAmelCase__		=    previous
													UpperCAmelCase__		=    node
													UpperCAmelCase__		=    self.rear
													node.freq += 1
													self._position_node(lowerCamelCase_				)






							def 					lowerCAmelCase     (						self :     Optional[Any]		, lowerCamelCase_ :     DoubleLinkedListNode[T, U]				):

													while node.prev is not None and node.prev.freq > node.freq:
																			# swap node with previous node
																			UpperCAmelCase__		=    node.prev

																			UpperCAmelCase__		=    previous_node.prev
																			UpperCAmelCase__		=    node.prev
																			UpperCAmelCase__		=    previous_node
																			UpperCAmelCase__		=    node






							def 					lowerCAmelCase     (						self :     Union[str, Any]		, lowerCamelCase_ :     DoubleLinkedListNode[T, U]				):

													if node.prev is None or node.next is None:
																			return None

													UpperCAmelCase__		=    node.next
													UpperCAmelCase__		=    node.prev
													UpperCAmelCase__		=    None
													UpperCAmelCase__		=    None
													return node




class 							lowerCamelCase       (			Generic[T, U] ):


							"""simple docstring"""
							SCREAMING_SNAKE_CASE										=      {}






							def __init__(						self :     str		, lowerCamelCase_ :     int				):
													UpperCAmelCase__		=    DoubleLinkedList()
													UpperCAmelCase__		=    capacity
													UpperCAmelCase__		=    0
													UpperCAmelCase__		=    0
													UpperCAmelCase__		=    0
													UpperCAmelCase__		=    {}






							def __repr__(						self :     Tuple				):
													return (
													    f"""CacheInfo(hits={self.hits}, misses={self.miss}, """
													    f"""capacity={self.capacity}, current_size={self.num_keys})"""
													)






							def __contains__(						self :     Optional[Any]		, lowerCamelCase_ :     T				):
													return key in self.cache






							def 					lowerCAmelCase     (						self :     Any		, lowerCamelCase_ :     T				):

													if key in self.cache:
																			self.hits += 1
																			UpperCAmelCase__		=    self.cache[key]
																			UpperCAmelCase__		=    self.list.remove(self.cache[key]				)
																			assert node == value_node

																			# node is guaranteed not None because it is in self.cache
																			assert node is not None
																			self.list.add(lowerCamelCase_				)
																			return node.val
													self.miss += 1
													return None






							def 					lowerCAmelCase     (						self :     Tuple		, lowerCamelCase_ :     T		, lowerCamelCase_ :     U				):

													if key not in self.cache:
																			if self.num_keys >= self.capacity:
																									# delete first node when over capacity
																									UpperCAmelCase__		=    self.list.head.next

																									# guaranteed to have a non-None first node when num_keys > 0
																									# explain to type checker via assertions
																									assert first_node is not None
																									assert first_node.key is not None
																									assert self.list.remove(lowerCamelCase_				) is not None
																									# first_node guaranteed to be in list

																									del self.cache[first_node.key]
																									self.num_keys -= 1
																			UpperCAmelCase__		=    DoubleLinkedListNode(lowerCamelCase_		, lowerCamelCase_				)
																			self.list.add(self.cache[key]				)
																			self.num_keys += 1

													else:
																			UpperCAmelCase__		=    self.list.remove(self.cache[key]				)
																			assert node is not None  # node guaranteed to be in list
																			UpperCAmelCase__		=    value
																			self.list.add(lowerCamelCase_				)






							@classmethod
							def 					lowerCAmelCase     (						cls :     type[LFUCache[T, U]]		, lowerCamelCase_ :     int = 128				):

													def cache_decorator_inner(lowerCamelCase_ :     Callable[[T], U]				) -> Callable[..., U]:
																			def cache_decorator_wrapper(*lowerCamelCase_ :     T				) -> U:
																									if func not in cls.decorator_function_to_instance_map:
																															UpperCAmelCase__		=    LFUCache(lowerCamelCase_				)

																									UpperCAmelCase__		=    cls.decorator_function_to_instance_map[func].get(args[0]				)
																									if result is None:
																															UpperCAmelCase__		=    func(*lowerCamelCase_				)
																															cls.decorator_function_to_instance_map[func].put(args[0]		, lowerCamelCase_				)
																									return result

																			def cache_info() -> LFUCache[T, U]:
																									return cls.decorator_function_to_instance_map[func]

																			setattr(lowerCamelCase_		, '''cache_info'''		, lowerCamelCase_				)  # noqa: B010

																			return cache_decorator_wrapper

													return cache_decorator_inner


if __name__ == "__main__":
						import doctest

						doctest.testmod()