


import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation

_SCREAMING_SNAKE_CASE :					List[str]			=	logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE :					Optional[int]			=	{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

_SCREAMING_SNAKE_CASE :					List[Any]			=	{
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

_SCREAMING_SNAKE_CASE :					List[str]			=	{
    """microsoft/deberta-base""": 512,
    """microsoft/deberta-large""": 512,
    """microsoft/deberta-xlarge""": 512,
    """microsoft/deberta-base-mnli""": 512,
    """microsoft/deberta-large-mnli""": 512,
    """microsoft/deberta-xlarge-mnli""": 512,
}

_SCREAMING_SNAKE_CASE :					List[Any]			=	{
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}







def 						_lowercase       (     )  ->   Optional[Any]:





     '''simple docstring'''





     UpperCamelCase__							:  str								=     (
         list(range(ord('''!'''			)		,ord('''~'''			) + 1			)			) + list(range(ord('''¡'''			)		,ord('''¬'''			) + 1			)			) + list(range(ord('''®'''			)		,ord('''ÿ'''			) + 1			)			)
     )
     UpperCamelCase__							:  List[Any]								=     bs[:]
     UpperCamelCase__							:  Dict								=     0
     for b in range(2**8			):
          if b not in bs:
               bs.append(__lowerCamelCase			)
               cs.append(2**8 + n			)
               n += 1
     UpperCamelCase__							:  Dict								=     [chr(__lowerCamelCase			) for n in cs]
     return dict(zip(__lowerCamelCase		,__lowerCamelCase			)			)







def 						_lowercase       (     __lowerCamelCase			:					Dict			)  ->   Tuple:





     '''simple docstring'''





     UpperCamelCase__							:  List[str]								=     set()
     UpperCamelCase__							:  int								=     word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)			)
          UpperCamelCase__							:  Any								=     char
     return pairs







class  UpperCamelCase__ ( __lowerCamelCase			):
      a__   :     int								=						VOCAB_FILES_NAMES
      a__   :     List[str]								=						PRETRAINED_VOCAB_FILES_MAP
      a__   :     Tuple								=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      a__   :     Any								=						['input_ids', 'attention_mask', 'token_type_ids']






      def __init__(     self					:   Dict,		__lowerCamelCase					:   List[str],		__lowerCamelCase					:   List[str],		__lowerCamelCase					:   str="replace",		__lowerCamelCase					:   str="[CLS]",		__lowerCamelCase					:   Optional[int]="[SEP]",		__lowerCamelCase					:   Union[str, Any]="[SEP]",		__lowerCamelCase					:   str="[CLS]",		__lowerCamelCase					:   List[Any]="[UNK]",		__lowerCamelCase					:   int="[PAD]",		__lowerCamelCase					:   List[Any]="[MASK]",		__lowerCamelCase					:   int=False,		__lowerCamelCase					:   str=False,		**__lowerCamelCase					:   int,		)  -> Dict:
           UpperCamelCase__							:  Union[str, Any]								=     AddedToken(__lowerCamelCase,		lstrip=__lowerCamelCase,		rstrip=__lowerCamelCase   ) if isinstance(__lowerCamelCase,		__lowerCamelCase   ) else bos_token
           UpperCamelCase__							:  Dict								=     AddedToken(__lowerCamelCase,		lstrip=__lowerCamelCase,		rstrip=__lowerCamelCase   ) if isinstance(__lowerCamelCase,		__lowerCamelCase   ) else eos_token
           UpperCamelCase__							:  Optional[Any]								=     AddedToken(__lowerCamelCase,		lstrip=__lowerCamelCase,		rstrip=__lowerCamelCase   ) if isinstance(__lowerCamelCase,		__lowerCamelCase   ) else sep_token
           UpperCamelCase__							:  List[Any]								=     AddedToken(__lowerCamelCase,		lstrip=__lowerCamelCase,		rstrip=__lowerCamelCase   ) if isinstance(__lowerCamelCase,		__lowerCamelCase   ) else cls_token
           UpperCamelCase__							:  Optional[int]								=     AddedToken(__lowerCamelCase,		lstrip=__lowerCamelCase,		rstrip=__lowerCamelCase   ) if isinstance(__lowerCamelCase,		__lowerCamelCase   ) else unk_token
           UpperCamelCase__							:  Dict								=     AddedToken(__lowerCamelCase,		lstrip=__lowerCamelCase,		rstrip=__lowerCamelCase   ) if isinstance(__lowerCamelCase,		__lowerCamelCase   ) else pad_token

           # Mask token behave like a normal word, i.e. include the space before it
           UpperCamelCase__							:  Optional[int]								=     AddedToken(__lowerCamelCase,		lstrip=__lowerCamelCase,		rstrip=__lowerCamelCase   ) if isinstance(__lowerCamelCase,		__lowerCamelCase   ) else mask_token

           super().__init__(
               errors=__lowerCamelCase,		bos_token=__lowerCamelCase,		eos_token=__lowerCamelCase,		unk_token=__lowerCamelCase,		sep_token=__lowerCamelCase,		cls_token=__lowerCamelCase,		pad_token=__lowerCamelCase,		mask_token=__lowerCamelCase,		add_prefix_space=__lowerCamelCase,		add_bos_token=__lowerCamelCase,		**__lowerCamelCase,		)
           UpperCamelCase__							:  Optional[int]								=     add_bos_token

           with open(__lowerCamelCase,		encoding='''utf-8'''   ) as vocab_handle:
                UpperCamelCase__							:  Any								=     json.load(__lowerCamelCase   )
           UpperCamelCase__							:  Optional[Any]								=     {v: k for k, v in self.encoder.items()}
           UpperCamelCase__							:  int								=     errors  # how to handle errors in decoding
           UpperCamelCase__							:  List[str]								=     bytes_to_unicode()
           UpperCamelCase__							:  Optional[int]								=     {v: k for k, v in self.byte_encoder.items()}
           with open(__lowerCamelCase,		encoding='''utf-8'''   ) as merges_handle:
                UpperCamelCase__							:  List[str]								=     merges_handle.read().split('''\n'''   )[1:-1]
           UpperCamelCase__							:  str								=     [tuple(merge.split()   ) for merge in bpe_merges]
           UpperCamelCase__							:  Optional[int]								=     dict(zip(__lowerCamelCase,		range(len(__lowerCamelCase   )   )   )   )
           UpperCamelCase__							:  Tuple								=     {}
           UpperCamelCase__							:  List[Any]								=     add_prefix_space

           # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
           UpperCamelCase__							:  List[str]								=     re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''   )






      @property
      # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
      def 	__lowercase(     self					:   Union[str, Any]   )  -> List[str]:
           return len(self.encoder   )






      def 	__lowercase(     self					:   Any   )  -> List[str]:
           return dict(self.encoder,		**self.added_tokens_encoder   )






      def 	__lowercase(     self					:   int,		__lowerCamelCase					:   List[Any]   )  -> Dict:
           if token in self.cache:
                return self.cache[token]
           UpperCamelCase__							:  int								=     tuple(__lowerCamelCase   )
           UpperCamelCase__							:  List[str]								=     get_pairs(__lowerCamelCase   )

           if not pairs:
                return token

           while True:
                UpperCamelCase__							:  Tuple								=     min(__lowerCamelCase,		key=lambda __lowerCamelCase   : self.bpe_ranks.get(__lowerCamelCase,		float('''inf'''   )   )   )
                if bigram not in self.bpe_ranks:
                     break
                UpperCamelCase__	,UpperCamelCase__							:  int								=     bigram
                UpperCamelCase__							:  Tuple								=     []
                UpperCamelCase__							:  Tuple								=     0
                while i < len(__lowerCamelCase   ):
                     try:
                          UpperCamelCase__							:  Tuple								=     word.index(__lowerCamelCase,		__lowerCamelCase   )
                     except ValueError:
                          new_word.extend(word[i:]   )
                          break
                     else:
                          new_word.extend(word[i:j]   )
                          UpperCamelCase__							:  Any								=     j

                     if word[i] == first and i < len(__lowerCamelCase   ) - 1 and word[i + 1] == second:
                          new_word.append(first + second   )
                          i += 2
                     else:
                          new_word.append(word[i]   )
                          i += 1
                UpperCamelCase__							:  List[str]								=     tuple(__lowerCamelCase   )
                UpperCamelCase__							:  Tuple								=     new_word
                if len(__lowerCamelCase   ) == 1:
                     break
                else:
                     UpperCamelCase__							:  Tuple								=     get_pairs(__lowerCamelCase   )
           UpperCamelCase__							:  Optional[int]								=     ''' '''.join(__lowerCamelCase   )
           UpperCamelCase__							:  Tuple								=     word
           return word






      def 	__lowercase(     self					:   Optional[int],		__lowerCamelCase					:   List[int],		__lowerCamelCase					:   Optional[List[int]] = None   )  -> List[int]:
           if token_ids_a is None:
                return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
           UpperCamelCase__							:  Dict								=     [self.cls_token_id]
           UpperCamelCase__							:  Tuple								=     [self.sep_token_id]
           return cls + token_ids_a + sep + token_ids_a + sep






      def 	__lowercase(     self					:   Optional[int],		__lowerCamelCase					:   List[int],		__lowerCamelCase					:   Optional[List[int]] = None,		__lowerCamelCase					:   bool = False   )  -> List[int]:
           if already_has_special_tokens:
                return super().get_special_tokens_mask(
                    token_ids_a=__lowerCamelCase,		token_ids_a=__lowerCamelCase,		already_has_special_tokens=__lowerCamelCase   )

           if token_ids_a is None:
                return [1] + ([0] * len(__lowerCamelCase   )) + [1]
           return [1] + ([0] * len(__lowerCamelCase   )) + [1] + ([0] * len(__lowerCamelCase   )) + [1]






      def 	__lowercase(     self					:   Optional[Any],		__lowerCamelCase					:   List[int],		__lowerCamelCase					:   Optional[List[int]] = None   )  -> List[int]:
           UpperCamelCase__							:  int								=     [self.sep_token_id]
           UpperCamelCase__							:  List[Any]								=     [self.cls_token_id]

           if token_ids_a is None:
                return len(cls + token_ids_a + sep   ) * [0]
           return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]






      def 	__lowercase(     self					:   List[str],		__lowerCamelCase					:   Dict   )  -> Dict:
           UpperCamelCase__							:  Dict								=     []
           for token in re.findall(self.pat,		__lowerCamelCase   ):
                UpperCamelCase__							:  int								=     ''''''.join(
                    self.byte_encoder[b] for b in token.encode('''utf-8'''   )   )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCamelCase   ).split(''' '''   )   )
           return bpe_tokens






      def 	__lowercase(     self					:   str,		__lowerCamelCase					:   Union[str, Any]   )  -> Union[str, Any]:
           return self.encoder.get(__lowerCamelCase,		self.encoder.get(self.unk_token   )   )






      def 	__lowercase(     self					:   Any,		__lowerCamelCase					:   Union[str, Any]   )  -> Tuple:
           return self.decoder.get(__lowerCamelCase   )






      def 	__lowercase(     self					:   List[Any],		__lowerCamelCase					:   List[str]   )  -> Dict:
           UpperCamelCase__							:  Union[str, Any]								=     ''''''.join(__lowerCamelCase   )
           UpperCamelCase__							:  List[str]								=     bytearray([self.byte_decoder[c] for c in text]   ).decode('''utf-8''',		errors=self.errors   )
           return text






      def 	__lowercase(     self					:   Optional[Any],		__lowerCamelCase					:   str,		__lowerCamelCase					:   Optional[str] = None   )  -> Tuple[str]:
           if not os.path.isdir(__lowerCamelCase   ):
                logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
                return
           UpperCamelCase__							:  Dict								=     os.path.join(
               __lowerCamelCase,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )
           UpperCamelCase__							:  str								=     os.path.join(
               __lowerCamelCase,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']   )

           with open(__lowerCamelCase,		'''w''',		encoding='''utf-8'''   ) as f:
                f.write(json.dumps(self.encoder,		indent=2,		sort_keys=__lowerCamelCase,		ensure_ascii=__lowerCamelCase   ) + '''\n'''   )

           UpperCamelCase__							:  Union[str, Any]								=     0
           with open(__lowerCamelCase,		'''w''',		encoding='''utf-8'''   ) as writer:
                writer.write('''#version: 0.2\n'''   )
                for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),		key=lambda __lowerCamelCase   : kv[1]   ):
                     if index != token_index:
                          logger.warning(
                              f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                              ''' Please check that the tokenizer is not corrupted!'''   )
                          UpperCamelCase__							:  List[Any]								=     token_index
                     writer.write(''' '''.join(__lowerCamelCase   ) + '''\n'''   )
                     index += 1

           return vocab_file, merge_file






      def 	__lowercase(     self					:   Optional[Any],		__lowerCamelCase					:   List[Any],		__lowerCamelCase					:   List[Any]=False,		**__lowerCamelCase					:   Any   )  -> Optional[Any]:
           UpperCamelCase__							:  Optional[Any]								=     kwargs.pop('''add_prefix_space''',		self.add_prefix_space   )
           if (is_split_into_words or add_prefix_space) and (len(__lowerCamelCase   ) > 0 and not text[0].isspace()):
                UpperCamelCase__							:  int								=     ''' ''' + text
           return (text, kwargs)






      def 	__lowercase(     self					:   str,		__lowerCamelCase					:   "Conversation"   )  -> List[int]:
           UpperCamelCase__							:  str								=     []
           for is_user, text in conversation.iter_texts():
                input_ids.extend(self.encode(__lowerCamelCase,		add_special_tokens=__lowerCamelCase   ) + [self.eos_token_id]   )
           if len(__lowerCamelCase   ) > self.model_max_length:
                UpperCamelCase__							:  List[Any]								=     input_ids[-self.model_max_length :]
           return input_ids








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


_SCREAMING_SNAKE_CASE :					str			=	{
    """configuration_donut_swin""": ["""DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP""", """DonutSwinConfig"""],
    """processing_donut""": ["""DonutProcessor"""],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _SCREAMING_SNAKE_CASE :					List[Any]			=	[
       """DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST""",
       """DonutSwinModel""",
       """DonutSwinPreTrainedModel""",
   ]

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _SCREAMING_SNAKE_CASE :					List[Any]			=	["""DonutFeatureExtractor"""]
   _SCREAMING_SNAKE_CASE :					Tuple			=	["""DonutImageProcessor"""]


if TYPE_CHECKING:
   from .configuration_donut_swin import DONUT_SWIN_PRETRAINED_CONFIG_ARCHIVE_MAP, DonutSwinConfig
   from .processing_donut import DonutProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_donut_swin import (
          DONUT_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
          DonutSwinModel,
          DonutSwinPreTrainedModel,
      )

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .feature_extraction_donut import DonutFeatureExtractor
      from .image_processing_donut import DonutImageProcessor

else:
   import sys

   _SCREAMING_SNAKE_CASE :					Optional[Any]			=	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




