



import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		UpperCAmelCase__			(			__UpperCamelCase						,unittest.TestCase   ):




			'''simple docstring'''






			UpperCamelCase									=       CLIPTokenizer
			UpperCamelCase									=       CLIPTokenizerFast
			UpperCamelCase									=       True
			UpperCamelCase									=       {}
			UpperCamelCase									=       False




			def  snake_case__     (   self							: Dict ):
					'''simple docstring'''





					super().setUp()

					# fmt: off
					__UpperCAmelCase    :     str													=	['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
					# fmt: on
					__UpperCAmelCase    :     Dict													=	dict(zip(a_       ,       range(len(a_ ) ) ) )
					__UpperCAmelCase    :     Optional[int]													=	['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
					__UpperCAmelCase    :     List[str]													=	{'''unk_token''': '''<unk>'''}

					__UpperCAmelCase    :     Dict													=	os.path.join(self.tmpdirname       ,       VOCAB_FILES_NAMES['''vocab_file'''] )
					__UpperCAmelCase    :     int													=	os.path.join(self.tmpdirname       ,       VOCAB_FILES_NAMES['''merges_file'''] )
					with open(self.vocab_file       ,       '''w'''       ,       encoding='''utf-8''' ) as fp:
							fp.write(json.dumps(a_ ) + '''\n''' )
					with open(self.merges_file       ,       '''w'''       ,       encoding='''utf-8''' ) as fp:
							fp.write('''\n'''.join(a_ ) )




			def  snake_case__     (   self							: Any       ,       **a_							: int ):
					'''simple docstring'''





					kwargs.update(self.special_tokens_map )
					return CLIPTokenizer.from_pretrained(self.tmpdirname       ,       **a_ )




			def  snake_case__     (   self							: str       ,       **a_							: Dict ):
					'''simple docstring'''





					kwargs.update(self.special_tokens_map )
					return CLIPTokenizerFast.from_pretrained(self.tmpdirname       ,       **a_ )




			def  snake_case__     (   self							: int       ,       a_							: Union[str, Any] ):
					'''simple docstring'''





					__UpperCAmelCase    :     Optional[Any]													=	'''lower newer'''
					__UpperCAmelCase    :     int													=	'''lower newer'''
					return input_text, output_text




			def  snake_case__     (   self							: List[Any] ):
					'''simple docstring'''





					__UpperCAmelCase    :     List[Any]													=	CLIPTokenizer(self.vocab_file       ,       self.merges_file       ,       **self.special_tokens_map )
					__UpperCAmelCase    :     List[Any]													=	'''lower newer'''
					__UpperCAmelCase    :     List[Any]													=	['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
					__UpperCAmelCase    :     Tuple													=	tokenizer.tokenize(a_ )
					self.assertListEqual(a_       ,       a_ )

					__UpperCAmelCase    :     Optional[Any]													=	tokens + [tokenizer.unk_token]
					__UpperCAmelCase    :     Any													=	[10, 2, 16, 9, 3, 2, 16, 20]
					self.assertListEqual(tokenizer.convert_tokens_to_ids(a_ )       ,       a_ )




			@require_ftfy
			def  snake_case__     (   self							: Union[str, Any] ):
					'''simple docstring'''





					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})' ):
									__UpperCAmelCase    :     Optional[Any]													=	self.tokenizer_class.from_pretrained(a_       ,       **a_ )
									__UpperCAmelCase    :     int													=	self.rust_tokenizer_class.from_pretrained(a_       ,       **a_ )

									__UpperCAmelCase    :     int													=	'''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
									__UpperCAmelCase    :     Any													=	tokenizer_s.tokenize(a_ )
									__UpperCAmelCase    :     Any													=	tokenizer_r.tokenize(a_ )

									self.assertListEqual(a_       ,       a_ )

									# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
									# with Tilde) encoded in 2 different ways
									__UpperCAmelCase    :     Optional[Any]													=	'''xa\u0303y''' + ''' ''' + '''x\xe3y'''
									__UpperCAmelCase    :     int													=	tokenizer_s.tokenize(a_ )
									__UpperCAmelCase    :     List[Any]													=	tokenizer_r.tokenize(a_ )

									self.assertListEqual(a_       ,       a_ )

									# Test that the tokenization is identical on unicode of space type
									__UpperCAmelCase    :     List[str]													=	[
									    '''\u0009''',  # (horizontal tab, '\t')
									    '''\u000B''',  # (vertical tab)
									    '''\u000C''',  # (form feed)
									    '''\u0020''',  # (space, ' ')
									    '''\u200E''',  # (left-to-right mark):w
									    '''\u200F''',  # (right-to-left mark)
									]
									for unicode_seq in spaces_unicodes:
											__UpperCAmelCase    :     List[Any]													=	tokenizer_s.tokenize(a_ )
											__UpperCAmelCase    :     Tuple													=	tokenizer_r.tokenize(a_ )

											self.assertListEqual(a_       ,       a_ )

									# Test that the tokenization is identical on unicode of line break type
									__UpperCAmelCase    :     Union[str, Any]													=	[
									    '''\u000A''',  # (line feed, '\n')
									    '''\r\n''',  # (carriage return and line feed, '\r\n')
									    '''\u000D''',  # (carriage return, '\r')
									    '''\r''',  # (carriage return, '\r')
									    '''\u000D''',  # (carriage return, '\r')
									    '''\u2028''',  # (line separator)
									    '''\u2029''',  # (paragraph separator)
									    # "\u0085", # (next line)
									]

									# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
									# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
									# space (and thus into an empty list).

									for unicode_seq in line_break_unicodes:
											__UpperCAmelCase    :     Dict													=	tokenizer_s.tokenize(a_ )
											__UpperCAmelCase    :     Any													=	tokenizer_r.tokenize(a_ )

											self.assertListEqual(a_       ,       a_ )




			def  snake_case__     (   self							: Dict ):
					'''simple docstring'''





					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})' ):
									__UpperCAmelCase    :     Any													=	'''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
									__UpperCAmelCase    :     List[Any]													=	F'{text_of_1_token} {text_of_1_token}'

									__UpperCAmelCase    :     Dict													=	self.rust_tokenizer_class.from_pretrained(
									    a_       ,       use_fast=a_       ,       )
									__UpperCAmelCase    :     int													=	tokenizer_r(a_       ,       return_offsets_mapping=a_       ,       add_special_tokens=a_ )
									self.assertEqual(encoding.offset_mapping[0]       ,       (0, len(a_ )) )
									self.assertEqual(
									    encoding.offset_mapping[1]       ,       (len(a_ ) + 1, len(a_ ) + 1 + len(a_ ))       ,       )

									__UpperCAmelCase    :     List[str]													=	F' {text}'

									__UpperCAmelCase    :     Union[str, Any]													=	self.rust_tokenizer_class.from_pretrained(
									    a_       ,       use_fast=a_       ,       )
									__UpperCAmelCase    :     List[str]													=	tokenizer_r(a_       ,       return_offsets_mapping=a_       ,       add_special_tokens=a_ )
									self.assertEqual(encoding.offset_mapping[0]       ,       (1, 1 + len(a_ )) )
									self.assertEqual(
									    encoding.offset_mapping[1]       ,       (1 + len(a_ ) + 1, 1 + len(a_ ) + 1 + len(a_ ))       ,       )




			def  snake_case__     (   self							: Any ):
					'''simple docstring'''





					with self.assertRaises(a_ ) as context:
							self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer''' )

					self.assertTrue(
					    context.exception.args[0].startswith(
					        '''The `backend_tokenizer` provided does not match the expected format.''' ) )




			@require_ftfy
			def  snake_case__     (   self							: Optional[int] ):
					'''simple docstring'''





					super().test_tokenization_python_rust_equals()




			def  snake_case__     (   self							: str ):
					'''simple docstring'''





					pass










from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


if TYPE_CHECKING:
				from ... import FeatureExtractionMixin, PreTrainedTokenizerBase, TensorType


__A   			=logging.get_logger(__name__)

__A   			={
    "microsoft/deberta-v2-xlarge": "https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/config.json",
    "microsoft/deberta-v2-xxlarge": "https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/config.json",
    "microsoft/deberta-v2-xlarge-mnli": (
        "https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/config.json"
    ),
    "microsoft/deberta-v2-xxlarge-mnli": (
        "https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/config.json"
    ),
}

class 		UpperCAmelCase__			(			__UpperCamelCase   ):




			'''simple docstring'''






			UpperCamelCase									=       """deberta-v2"""




			def __init__(   self							: Optional[int]       ,       a_							: List[str]=12_81_00       ,       a_							: Optional[Any]=15_36       ,       a_							: Optional[Any]=24       ,       a_							: List[Any]=24       ,       a_							: Optional[int]=61_44       ,       a_							: List[Any]="gelu"       ,       a_							: Any=0.1       ,       a_							: Tuple=0.1       ,       a_							: Optional[Any]=5_12       ,       a_							: Tuple=0       ,       a_							: Dict=0.0_2       ,       a_							: Optional[Any]=1e-7       ,       a_							: List[str]=False       ,       a_							: List[Any]=-1       ,       a_							: List[str]=0       ,       a_							: Optional[Any]=True       ,       a_							: List[Any]=None       ,       a_							: Optional[int]=0       ,       a_							: Tuple="gelu"       ,       **a_							: List[str]       ,       ):
					'''simple docstring'''





					super().__init__(**a_ )

					__UpperCAmelCase    :     Any													=	hidden_size
					__UpperCAmelCase    :     Any													=	num_hidden_layers
					__UpperCAmelCase    :     List[Any]													=	num_attention_heads
					__UpperCAmelCase    :     Union[str, Any]													=	intermediate_size
					__UpperCAmelCase    :     List[str]													=	hidden_act
					__UpperCAmelCase    :     int													=	hidden_dropout_prob
					__UpperCAmelCase    :     List[str]													=	attention_probs_dropout_prob
					__UpperCAmelCase    :     str													=	max_position_embeddings
					__UpperCAmelCase    :     int													=	type_vocab_size
					__UpperCAmelCase    :     Tuple													=	initializer_range
					__UpperCAmelCase    :     Optional[int]													=	relative_attention
					__UpperCAmelCase    :     int													=	max_relative_positions
					__UpperCAmelCase    :     Any													=	pad_token_id
					__UpperCAmelCase    :     int													=	position_biased_input

					# Backwards compatibility
					if type(a_ ) == str:
							__UpperCAmelCase    :     Optional[Any]													=	[x.strip() for x in pos_att_type.lower().split('''|''' )]

					__UpperCAmelCase    :     Tuple													=	pos_att_type
					__UpperCAmelCase    :     int													=	vocab_size
					__UpperCAmelCase    :     Optional[Any]													=	layer_norm_eps

					__UpperCAmelCase    :     str													=	kwargs.get('''pooler_hidden_size'''       ,       a_ )
					__UpperCAmelCase    :     Union[str, Any]													=	pooler_dropout
					__UpperCAmelCase    :     Tuple													=	pooler_hidden_act

class 		UpperCAmelCase__			(			__UpperCamelCase   ):




			'''simple docstring'''






			@property
			def  snake_case__     (   self							: Optional[Any] ):
					'''simple docstring'''





					if self.task == "multiple-choice":
							__UpperCAmelCase    :     List[Any]													=	{0: '''batch''', 1: '''choice''', 2: '''sequence'''}
					else:
							__UpperCAmelCase    :     int													=	{0: '''batch''', 1: '''sequence'''}
					if self._config.type_vocab_size > 0:
							return OrderedDict(
							    [('''input_ids''', dynamic_axis), ('''attention_mask''', dynamic_axis), ('''token_type_ids''', dynamic_axis)] )
					else:
							return OrderedDict([('''input_ids''', dynamic_axis), ('''attention_mask''', dynamic_axis)] )




			@property
			def  snake_case__     (   self							: Union[str, Any] ):
					'''simple docstring'''





					return 12




			def  snake_case__     (   self							: Any       ,       a_							: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]       ,       a_							: int = -1       ,       a_							: int = -1       ,       a_							: int = -1       ,       a_							: bool = False       ,       a_							: Optional["TensorType"] = None       ,       a_							: int = 3       ,       a_							: int = 40       ,       a_							: int = 40       ,       a_							: "PreTrainedTokenizerBase" = None       ,       ):
					'''simple docstring'''





					__UpperCAmelCase    :     List[Any]													=	super().generate_dummy_inputs(preprocessor=a_       ,       framework=a_ )
					if self._config.type_vocab_size == 0 and "token_type_ids" in dummy_inputs:
							del dummy_inputs["token_type_ids"]
					return dummy_inputs





