




'''simple docstring'''

from typing import TYPE_CHECKING

from ..utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_tf_available, is_torch_available


A_        =   {
    "configuration_utils": ["GenerationConfig"],
    "streamers": ["TextIteratorStreamer", "TextStreamer"],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A_        =   [
						    "Constraint",
						    "ConstraintListState",
						    "DisjunctiveConstraint",
						    "PhrasalConstraint",
						]
						A_        =   [
						    "BeamHypotheses",
						    "BeamScorer",
						    "BeamSearchScorer",
						    "ConstrainedBeamSearchScorer",
						]
						A_        =   [
						    "EpsilonLogitsWarper",
						    "EtaLogitsWarper",
						    "ForcedBOSTokenLogitsProcessor",
						    "ForcedEOSTokenLogitsProcessor",
						    "HammingDiversityLogitsProcessor",
						    "InfNanRemoveLogitsProcessor",
						    "LogitsProcessor",
						    "LogitsProcessorList",
						    "LogitsWarper",
						    "MinLengthLogitsProcessor",
						    "MinNewTokensLengthLogitsProcessor",
						    "NoBadWordsLogitsProcessor",
						    "NoRepeatNGramLogitsProcessor",
						    "PrefixConstrainedLogitsProcessor",
						    "RepetitionPenaltyLogitsProcessor",
						    "SequenceBiasLogitsProcessor",
						    "EncoderRepetitionPenaltyLogitsProcessor",
						    "TemperatureLogitsWarper",
						    "TopKLogitsWarper",
						    "TopPLogitsWarper",
						    "TypicalLogitsWarper",
						    "EncoderNoRepeatNGramLogitsProcessor",
						    "ExponentialDecayLengthPenalty",
						    "LogitNormalization",
						]
						A_        =   [
						    "MaxNewTokensCriteria",
						    "MaxLengthCriteria",
						    "MaxTimeCriteria",
						    "StoppingCriteria",
						    "StoppingCriteriaList",
						    "validate_stopping_criteria",
						]
						A_        =   [
						    "GenerationMixin",
						    "top_k_top_p_filtering",
						    "GreedySearchEncoderDecoderOutput",
						    "GreedySearchDecoderOnlyOutput",
						    "SampleEncoderDecoderOutput",
						    "SampleDecoderOnlyOutput",
						    "BeamSearchEncoderDecoderOutput",
						    "BeamSearchDecoderOnlyOutput",
						    "BeamSampleEncoderDecoderOutput",
						    "BeamSampleDecoderOnlyOutput",
						    "ContrastiveSearchEncoderDecoderOutput",
						    "ContrastiveSearchDecoderOnlyOutput",
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A_        =   [
						    "TFForcedBOSTokenLogitsProcessor",
						    "TFForcedEOSTokenLogitsProcessor",
						    "TFLogitsProcessor",
						    "TFLogitsProcessorList",
						    "TFLogitsWarper",
						    "TFMinLengthLogitsProcessor",
						    "TFNoBadWordsLogitsProcessor",
						    "TFNoRepeatNGramLogitsProcessor",
						    "TFRepetitionPenaltyLogitsProcessor",
						    "TFTemperatureLogitsWarper",
						    "TFTopKLogitsWarper",
						    "TFTopPLogitsWarper",
						    "TFForceTokensLogitsProcessor",
						    "TFSuppressTokensAtBeginLogitsProcessor",
						    "TFSuppressTokensLogitsProcessor",
						]
						A_        =   [
						    "TFGenerationMixin",
						    "tf_top_k_top_p_filtering",
						    "TFGreedySearchDecoderOnlyOutput",
						    "TFGreedySearchEncoderDecoderOutput",
						    "TFSampleEncoderDecoderOutput",
						    "TFSampleDecoderOnlyOutput",
						    "TFBeamSearchEncoderDecoderOutput",
						    "TFBeamSearchDecoderOnlyOutput",
						    "TFBeamSampleEncoderDecoderOutput",
						    "TFBeamSampleDecoderOnlyOutput",
						    "TFContrastiveSearchEncoderDecoderOutput",
						    "TFContrastiveSearchDecoderOnlyOutput",
						]

try:
						if not is_flax_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A_        =   [
						    "FlaxForcedBOSTokenLogitsProcessor",
						    "FlaxForcedEOSTokenLogitsProcessor",
						    "FlaxLogitsProcessor",
						    "FlaxLogitsProcessorList",
						    "FlaxLogitsWarper",
						    "FlaxMinLengthLogitsProcessor",
						    "FlaxTemperatureLogitsWarper",
						    "FlaxTopKLogitsWarper",
						    "FlaxTopPLogitsWarper",
						]
						A_        =   [
						    "FlaxGenerationMixin",
						    "FlaxGreedySearchOutput",
						    "FlaxSampleOutput",
						    "FlaxBeamSearchOutput",
						]

if TYPE_CHECKING:
						from .configuration_utils import GenerationConfig
						from .streamers import TextIteratorStreamer, TextStreamer

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .beam_constraints import Constraint, ConstraintListState, DisjunctiveConstraint, PhrasalConstraint
												from .beam_search import BeamHypotheses, BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer
												from .logits_process import (
												    EncoderNoRepeatNGramLogitsProcessor,
												    EncoderRepetitionPenaltyLogitsProcessor,
												    EpsilonLogitsWarper,
												    EtaLogitsWarper,
												    ExponentialDecayLengthPenalty,
												    ForcedBOSTokenLogitsProcessor,
												    ForcedEOSTokenLogitsProcessor,
												    HammingDiversityLogitsProcessor,
												    InfNanRemoveLogitsProcessor,
												    LogitNormalization,
												    LogitsProcessor,
												    LogitsProcessorList,
												    LogitsWarper,
												    MinLengthLogitsProcessor,
												    MinNewTokensLengthLogitsProcessor,
												    NoBadWordsLogitsProcessor,
												    NoRepeatNGramLogitsProcessor,
												    PrefixConstrainedLogitsProcessor,
												    RepetitionPenaltyLogitsProcessor,
												    SequenceBiasLogitsProcessor,
												    TemperatureLogitsWarper,
												    TopKLogitsWarper,
												    TopPLogitsWarper,
												    TypicalLogitsWarper,
												)
												from .stopping_criteria import (
												    MaxLengthCriteria,
												    MaxNewTokensCriteria,
												    MaxTimeCriteria,
												    StoppingCriteria,
												    StoppingCriteriaList,
												    validate_stopping_criteria,
												)
												from .utils import (
												    BeamSampleDecoderOnlyOutput,
												    BeamSampleEncoderDecoderOutput,
												    BeamSearchDecoderOnlyOutput,
												    BeamSearchEncoderDecoderOutput,
												    ContrastiveSearchDecoderOnlyOutput,
												    ContrastiveSearchEncoderDecoderOutput,
												    GenerationMixin,
												    GreedySearchDecoderOnlyOutput,
												    GreedySearchEncoderDecoderOutput,
												    SampleDecoderOnlyOutput,
												    SampleEncoderDecoderOutput,
												    top_k_top_p_filtering,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tf_logits_process import (
												    TFForcedBOSTokenLogitsProcessor,
												    TFForcedEOSTokenLogitsProcessor,
												    TFForceTokensLogitsProcessor,
												    TFLogitsProcessor,
												    TFLogitsProcessorList,
												    TFLogitsWarper,
												    TFMinLengthLogitsProcessor,
												    TFNoBadWordsLogitsProcessor,
												    TFNoRepeatNGramLogitsProcessor,
												    TFRepetitionPenaltyLogitsProcessor,
												    TFSuppressTokensAtBeginLogitsProcessor,
												    TFSuppressTokensLogitsProcessor,
												    TFTemperatureLogitsWarper,
												    TFTopKLogitsWarper,
												    TFTopPLogitsWarper,
												)
												from .tf_utils import (
												    TFBeamSampleDecoderOnlyOutput,
												    TFBeamSampleEncoderDecoderOutput,
												    TFBeamSearchDecoderOnlyOutput,
												    TFBeamSearchEncoderDecoderOutput,
												    TFContrastiveSearchDecoderOnlyOutput,
												    TFContrastiveSearchEncoderDecoderOutput,
												    TFGenerationMixin,
												    TFGreedySearchDecoderOnlyOutput,
												    TFGreedySearchEncoderDecoderOutput,
												    TFSampleDecoderOnlyOutput,
												    TFSampleEncoderDecoderOutput,
												    tf_top_k_top_p_filtering,
												)

						try:
												if not is_flax_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .flax_logits_process import (
												    FlaxForcedBOSTokenLogitsProcessor,
												    FlaxForcedEOSTokenLogitsProcessor,
												    FlaxLogitsProcessor,
												    FlaxLogitsProcessorList,
												    FlaxLogitsWarper,
												    FlaxMinLengthLogitsProcessor,
												    FlaxTemperatureLogitsWarper,
												    FlaxTopKLogitsWarper,
												    FlaxTopPLogitsWarper,
												)
												from .flax_utils import FlaxBeamSearchOutput, FlaxGenerationMixin, FlaxGreedySearchOutput, FlaxSampleOutput
else:
						import sys

						A_        =   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






'''simple docstring'''

import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_deberta import DebertaTokenizer


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation


A_        =   logging.get_logger(__name__)

A_        =   {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

A_        =   {
    "vocab_file": {
        "microsoft/deberta-base": "https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json",
        "microsoft/deberta-large": "https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json",
        "microsoft/deberta-xlarge": "https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json",
        "microsoft/deberta-base-mnli": "https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json",
        "microsoft/deberta-large-mnli": "https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json",
        "microsoft/deberta-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "microsoft/deberta-base": "https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt",
        "microsoft/deberta-large": "https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt",
        "microsoft/deberta-xlarge": "https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt",
        "microsoft/deberta-base-mnli": "https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt",
        "microsoft/deberta-large-mnli": "https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt",
        "microsoft/deberta-xlarge-mnli": (
            "https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"
        ),
    },
}

A_        =   {
    "microsoft/deberta-base": 512,
    "microsoft/deberta-large": 512,
    "microsoft/deberta-xlarge": 512,
    "microsoft/deberta-base-mnli": 512,
    "microsoft/deberta-large-mnli": 512,
    "microsoft/deberta-xlarge-mnli": 512,
}

A_        =   {
    "microsoft/deberta-base": {"do_lower_case": False},
    "microsoft/deberta-large": {"do_lower_case": False},
}




class 		SCREAMING_SNAKE_CASE_				(_UpperCAmelCase			):



			"""simple docstring"""






			__magic_name__   :							Optional[Any]             =					VOCAB_FILES_NAMES
			__magic_name__   :							int             =					PRETRAINED_VOCAB_FILES_MAP
			__magic_name__   :							Dict             =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__magic_name__   :							List[str]             =					["""input_ids""", """attention_mask""", """token_type_ids"""]
			__magic_name__   :							Optional[int]             =					DebertaTokenizer



			def __init__(       self							, snake_case_=None							, snake_case_=None							, snake_case_=None							, snake_case_="replace"							, snake_case_="[CLS]"							, snake_case_="[SEP]"							, snake_case_="[SEP]"							, snake_case_="[CLS]"							, snake_case_="[UNK]"							, snake_case_="[PAD]"							, snake_case_="[MASK]"							, snake_case_=False							, **snake_case_							, ):
									super().__init__(
									    snake_case_							, snake_case_							, tokenizer_file=snake_case_							, errors=snake_case_							, bos_token=snake_case_							, eos_token=snake_case_							, unk_token=snake_case_							, sep_token=snake_case_							, cls_token=snake_case_							, pad_token=snake_case_							, mask_token=snake_case_							, add_prefix_space=snake_case_							, **snake_case_							, )
									snake_case   :					List[str]							=						kwargs.pop("add_bos_token"							, snake_case_				)

									snake_case   :					int							=						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
									if pre_tok_state.get("add_prefix_space"							, snake_case_				) != add_prefix_space:
															snake_case   :					Dict							=						getattr(snake_case_							, pre_tok_state.pop("type"				)				)
															snake_case   :					int							=						add_prefix_space
															snake_case   :					Optional[int]							=						pre_tok_class(**snake_case_				)

									snake_case   :					Any							=						add_prefix_space



			@property
			def      lowerCamelCase   (       self				):
									if self._mask_token is None:
															if self.verbose:
																					logger.error("Using mask_token, but it is not set yet."				)
															return None
									return str(self._mask_token				)



			@mask_token.setter
			def      lowerCamelCase   (       self							, snake_case_				):
									snake_case   :					Optional[Any]							=						AddedToken(snake_case_							, lstrip=snake_case_							, rstrip=snake_case_				) if isinstance(snake_case_							, snake_case_				) else value
									snake_case   :					Optional[int]							=						value



			def      lowerCamelCase   (       self							, snake_case_							, snake_case_ = None				):
									if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									snake_case   :					Optional[int]							=						[self.cls_token_id]
									snake_case   :					Optional[int]							=						[self.sep_token_id]
									return cls + token_ids_a + sep + token_ids_a + sep



			def      lowerCamelCase   (       self							, snake_case_							, snake_case_ = None				):
									snake_case   :					Dict							=						[self.sep_token_id]
									snake_case   :					Dict							=						[self.cls_token_id]

									if token_ids_a is None:
															return len(cls + token_ids_a + sep				) * [0]
									return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]



			def      lowerCamelCase   (       self							, *snake_case_							, **snake_case_				):
									snake_case   :					str							=						kwargs.get("is_split_into_words"							, snake_case_				)
									assert self.add_prefix_space or not is_split_into_words, (
									    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
									    "to use it with pretokenized inputs."
									)

									return super()._batch_encode_plus(*snake_case_							, **snake_case_				)



			def      lowerCamelCase   (       self							, *snake_case_							, **snake_case_				):
									snake_case   :					List[Any]							=						kwargs.get("is_split_into_words"							, snake_case_				)

									assert self.add_prefix_space or not is_split_into_words, (
									    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
									    "to use it with pretokenized inputs."
									)

									return super()._encode_plus(*snake_case_							, **snake_case_				)



			def      lowerCamelCase   (       self							, snake_case_							, snake_case_ = None				):
									snake_case   :					List[str]							=						self._tokenizer.model.save(snake_case_							, name=snake_case_				)
									return tuple(snake_case_				)






			def      lowerCamelCase   (       self							, snake_case_				):
									snake_case   :					int							=						[]
									for is_user, text in conversation.iter_texts():
															input_ids.extend(self.encode(snake_case_							, add_special_tokens=snake_case_				) + [self.eos_token_id]				)

									if len(snake_case_				) > self.model_max_length:
															snake_case   :					Union[str, Any]							=						input_ids[-self.model_max_length :]
									return input_ids





