
"""simple docstring"""
import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation

_a   :      Tuple       =				logging.get_logger(__name__)

_a   :      List[str]       =				{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

_a   :      List[Any]       =				{
    """vocab_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """microsoft/deberta-base""": """https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt""",
        """microsoft/deberta-large""": """https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge""": """https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt""",
        """microsoft/deberta-base-mnli""": """https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-large-mnli""": """https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt""",
        """microsoft/deberta-xlarge-mnli""": (
            """https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt"""
        ),
    },
}

_a   :      Union[str, Any]       =				{
    """microsoft/deberta-base""": 512,
    """microsoft/deberta-large""": 512,
    """microsoft/deberta-xlarge""": 512,
    """microsoft/deberta-base-mnli""": 512,
    """microsoft/deberta-large-mnli""": 512,
    """microsoft/deberta-xlarge-mnli""": 512,
}

_a   :      Dict       =				{
    """microsoft/deberta-base""": {"""do_lower_case""": False},
    """microsoft/deberta-large""": {"""do_lower_case""": False},
}
def 					a__ ( ):

  """simple docstring"""







  _snake_case  :    Union[str, Any]            = (
      list(range(ord("!"		)							,  ord("~"		) + 1		)		) + list(range(ord("¡"		)							,  ord("¬"		) + 1		)		) + list(range(ord("®"		)							,  ord("ÿ"		) + 1		)		)
  )
  _snake_case  :    int            = bs[:]
  _snake_case  :    Optional[Any]            = 0
  for b in range(2**8		):
    if b not in bs:
      bs.append(a		)
      cs.append(2**8 + n		)
      n += 1
  _snake_case  :    Dict            = [chr(a		) for n in cs]
  return dict(zip(a							,  a		)		)
def 					a__ ( a :					Optional[int]		):

  """simple docstring"""







  _snake_case  :    str            = set()
  _snake_case  :    List[str]            = word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)		)
    _snake_case  :    Union[str, Any]            = char
  return pairs






class   _UpperCAmelCase					(       _snake_case):
    __lowercase				:			Optional[int]	=       VOCAB_FILES_NAMES
    __lowercase				:			List[Any]	=       PRETRAINED_VOCAB_FILES_MAP
    __lowercase				:			Optional[int]	=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowercase				:			Dict	=       ["""input_ids""", """attention_mask""", """token_type_ids"""]
    def __init__(     self    ,     snake_case_    ,     snake_case_    ,     snake_case_="replace"    ,     snake_case_="[CLS]"    ,     snake_case_="[SEP]"    ,     snake_case_="[SEP]"    ,     snake_case_="[CLS]"    ,     snake_case_="[UNK]"    ,     snake_case_="[PAD]"    ,     snake_case_="[MASK]"    ,     snake_case_=False    ,     snake_case_=False    ,     **snake_case_    ,     ):
      _snake_case  :    Tuple            = AddedToken(snake_case_    ,     lstrip=snake_case_    ,     rstrip=snake_case_  ) if isinstance(snake_case_    ,     snake_case_  ) else bos_token
      _snake_case  :    Optional[Any]            = AddedToken(snake_case_    ,     lstrip=snake_case_    ,     rstrip=snake_case_  ) if isinstance(snake_case_    ,     snake_case_  ) else eos_token
      _snake_case  :    Dict            = AddedToken(snake_case_    ,     lstrip=snake_case_    ,     rstrip=snake_case_  ) if isinstance(snake_case_    ,     snake_case_  ) else sep_token
      _snake_case  :    Optional[Any]            = AddedToken(snake_case_    ,     lstrip=snake_case_    ,     rstrip=snake_case_  ) if isinstance(snake_case_    ,     snake_case_  ) else cls_token
      _snake_case  :    Dict            = AddedToken(snake_case_    ,     lstrip=snake_case_    ,     rstrip=snake_case_  ) if isinstance(snake_case_    ,     snake_case_  ) else unk_token
      _snake_case  :    List[str]            = AddedToken(snake_case_    ,     lstrip=snake_case_    ,     rstrip=snake_case_  ) if isinstance(snake_case_    ,     snake_case_  ) else pad_token

      # Mask token behave like a normal word, i.e. include the space before it
      _snake_case  :    str            = AddedToken(snake_case_    ,     lstrip=snake_case_    ,     rstrip=snake_case_  ) if isinstance(snake_case_    ,     snake_case_  ) else mask_token

      super().__init__(
          errors=snake_case_    ,     bos_token=snake_case_    ,     eos_token=snake_case_    ,     unk_token=snake_case_    ,     sep_token=snake_case_    ,     cls_token=snake_case_    ,     pad_token=snake_case_    ,     mask_token=snake_case_    ,     add_prefix_space=snake_case_    ,     add_bos_token=snake_case_    ,     **snake_case_    ,     )
      _snake_case  :    List[Any]            = add_bos_token

      with open(snake_case_    ,     encoding="utf-8"  ) as vocab_handle:
        _snake_case  :    Dict            = json.load(snake_case_  )
      _snake_case  :    Tuple            = {v: k for k, v in self.encoder.items()}
      _snake_case  :    str            = errors  # how to handle errors in decoding
      _snake_case  :    Optional[Any]            = bytes_to_unicode()
      _snake_case  :    str            = {v: k for k, v in self.byte_encoder.items()}
      with open(snake_case_    ,     encoding="utf-8"  ) as merges_handle:
        _snake_case  :    Union[str, Any]            = merges_handle.read().split("\n"  )[1:-1]
      _snake_case  :    List[str]            = [tuple(merge.split()  ) for merge in bpe_merges]
      _snake_case  :    Any            = dict(zip(snake_case_    ,     range(len(snake_case_  )  )  )  )
      _snake_case  :    Optional[int]            = {}
      _snake_case  :    Dict            = add_prefix_space

      # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
      _snake_case  :    Optional[Any]            = re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"  )
    @property
    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
    def        lowerCamelCase__						(     self  ):
      return len(self.encoder  )
    def        lowerCamelCase__						(     self  ):
      return dict(self.encoder    ,     **self.added_tokens_encoder  )
    def        lowerCamelCase__						(     self    ,     snake_case_  ):
      if token in self.cache:
        return self.cache[token]
      _snake_case  :    Optional[Any]            = tuple(snake_case_  )
      _snake_case  :    List[str]            = get_pairs(snake_case_  )

      if not pairs:
        return token

      while True:
        _snake_case  :    Optional[int]            = min(snake_case_    ,     key=lambda snake_case_  : self.bpe_ranks.get(snake_case_    ,     float("inf"  )  )  )
        if bigram not in self.bpe_ranks:
          break
        _snake_case      ,       _snake_case  :    Tuple            = bigram
        _snake_case  :    List[str]            = []
        _snake_case  :    Optional[int]            = 0
        while i < len(snake_case_  ):
          try:
            _snake_case  :    Union[str, Any]            = word.index(snake_case_    ,     snake_case_  )
          except ValueError:
            new_word.extend(word[i:]  )
            break
          else:
            new_word.extend(word[i:j]  )
            _snake_case  :    Optional[int]            = j

          if word[i] == first and i < len(snake_case_  ) - 1 and word[i + 1] == second:
            new_word.append(first + second  )
            i += 2
          else:
            new_word.append(word[i]  )
            i += 1
        _snake_case  :    Tuple            = tuple(snake_case_  )
        _snake_case  :    str            = new_word
        if len(snake_case_  ) == 1:
          break
        else:
          _snake_case  :    Optional[Any]            = get_pairs(snake_case_  )
      _snake_case  :    Dict            = " ".join(snake_case_  )
      _snake_case  :    Any            = word
      return word
    def        lowerCamelCase__						(     self    ,     snake_case_    ,     snake_case_ = None  ):
      if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      _snake_case  :    int            = [self.cls_token_id]
      _snake_case  :    Optional[int]            = [self.sep_token_id]
      return cls + token_ids_a + sep + token_ids_a + sep
    def        lowerCamelCase__						(     self    ,     snake_case_    ,     snake_case_ = None    ,     snake_case_ = False  ):
      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=snake_case_    ,     token_ids_a=snake_case_    ,     already_has_special_tokens=snake_case_  )

      if token_ids_a is None:
        return [1] + ([0] * len(snake_case_  )) + [1]
      return [1] + ([0] * len(snake_case_  )) + [1] + ([0] * len(snake_case_  )) + [1]
    def        lowerCamelCase__						(     self    ,     snake_case_    ,     snake_case_ = None  ):
      _snake_case  :    List[Any]            = [self.sep_token_id]
      _snake_case  :    List[Any]            = [self.cls_token_id]

      if token_ids_a is None:
        return len(cls + token_ids_a + sep  ) * [0]
      return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]
    def        lowerCamelCase__						(     self    ,     snake_case_  ):
      _snake_case  :    str            = []
      for token in re.findall(self.pat    ,     snake_case_  ):
        _snake_case  :    List[str]            = "".join(
            self.byte_encoder[b] for b in token.encode("utf-8"  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
        bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case_  ).split(" "  )  )
      return bpe_tokens
    def        lowerCamelCase__						(     self    ,     snake_case_  ):
      return self.encoder.get(snake_case_    ,     self.encoder.get(self.unk_token  )  )
    def        lowerCamelCase__						(     self    ,     snake_case_  ):
      return self.decoder.get(snake_case_  )
    def        lowerCamelCase__						(     self    ,     snake_case_  ):
      _snake_case  :    List[Any]            = "".join(snake_case_  )
      _snake_case  :    Optional[Any]            = bytearray([self.byte_decoder[c] for c in text]  ).decode("utf-8"    ,     errors=self.errors  )
      return text
    def        lowerCamelCase__						(     self    ,     snake_case_    ,     snake_case_ = None  ):
      if not os.path.isdir(snake_case_  ):
        logger.error(F'Vocabulary path ({save_directory}) should be a directory'  )
        return
      _snake_case  :    Tuple            = os.path.join(
          snake_case_    ,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )
      _snake_case  :    Union[str, Any]            = os.path.join(
          snake_case_    ,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]  )

      with open(snake_case_    ,     "w"    ,     encoding="utf-8"  ) as f:
        f.write(json.dumps(self.encoder    ,     indent=2    ,     sort_keys=snake_case_    ,     ensure_ascii=snake_case_  ) + "\n"  )

      _snake_case  :    Union[str, Any]            = 0
      with open(snake_case_    ,     "w"    ,     encoding="utf-8"  ) as writer:
        writer.write("#version: 0.2\n"  )
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,     key=lambda snake_case_  : kv[1]  ):
          if index != token_index:
            logger.warning(
                F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
                " Please check that the tokenizer is not corrupted!"  )
            _snake_case  :    List[str]            = token_index
          writer.write(" ".join(snake_case_  ) + "\n"  )
          index += 1

      return vocab_file, merge_file
    def        lowerCamelCase__						(     self    ,     snake_case_    ,     snake_case_=False    ,     **snake_case_  ):
      _snake_case  :    List[str]            = kwargs.pop("add_prefix_space"    ,     self.add_prefix_space  )
      if (is_split_into_words or add_prefix_space) and (len(snake_case_  ) > 0 and not text[0].isspace()):
        _snake_case  :    List[Any]            = " " + text
      return (text, kwargs)


    def        lowerCamelCase__						(     self    ,     snake_case_  ):
      _snake_case  :    Any            = []
      for is_user, text in conversation.iter_texts():
        input_ids.extend(self.encode(snake_case_    ,     add_special_tokens=snake_case_  ) + [self.eos_token_id]  )
      if len(snake_case_  ) > self.model_max_length:
        _snake_case  :    str            = input_ids[-self.model_max_length :]
      return input_ids









"""simple docstring"""


import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer


class    snake_case						(				tf.keras.layers.Layer		):


   '''simple docstring'''

   def __init__(      self					:	List[str],			_lowerCamelCase					:	List,			_lowerCamelCase					:	bool,			_lowerCamelCase					:	int = None,			_lowerCamelCase					:	int = None,			_lowerCamelCase					:	int = None,			_lowerCamelCase					:	str = "longest",			_lowerCamelCase					:	bool = True,			_lowerCamelCase					:	int = 5_12,			_lowerCamelCase					:	int = None,			_lowerCamelCase					:	bool = True,			_lowerCamelCase					:	bool = True,			_lowerCamelCase					:	bool = True,			**_lowerCamelCase					:	int,			):

     '''simple docstring'''
     super().__init__()
     if use_fast_bert_tokenizer:
       __A		     =					FastBertTokenizer(
           _lowerCamelCase,			token_out_type=tf.intaa,			lower_case_nfd_strip_accents=_lowerCamelCase,			**_lowerCamelCase    )
     else:
       __A		     =					tf.lookup.StaticVocabularyTable(
           tf.lookup.KeyValueTensorInitializer(
               keys=_lowerCamelCase,			key_dtype=tf.string,			values=tf.range(tf.size(_lowerCamelCase,			out_type=tf.intaa    ),			dtype=tf.intaa    ),			value_dtype=tf.intaa,			),			num_oov_buckets=1,			)
       __A		     =					BertTokenizerLayer(
           _lowerCamelCase,			token_out_type=tf.intaa,			lower_case=_lowerCamelCase,			**_lowerCamelCase    )

     __A		     =					vocab_list
     __A		     =					do_lower_case
     __A		     =					cls_token_id or vocab_list.index('''[CLS]'''    )
     __A		     =					sep_token_id or vocab_list.index('''[SEP]'''    )
     __A		     =					pad_token_id or vocab_list.index('''[PAD]'''    )
     __A		     =					ShrinkLongestTrimmer(max_length - 3,			axis=1    )  # Allow room for special tokens
     __A		     =					max_length
     __A		     =					padding
     __A		     =					truncation
     __A		     =					pad_to_multiple_of
     __A		     =					return_token_type_ids
     __A		     =					return_attention_mask


   @classmethod
   def 							_SCREAMING_SNAKE_CASE							(      cls					:	int,			_lowerCamelCase					:	"PreTrainedTokenizerBase",			**_lowerCamelCase					:	List[Any]    ):  # noqa: F821

     '''simple docstring'''
     __A		     =					kwargs.pop('''do_lower_case''',			_lowerCamelCase    )
     __A		     =					tokenizer.do_lower_case if do_lower_case is None else do_lower_case
     __A		     =					kwargs.pop('''cls_token_id''',			_lowerCamelCase    )
     __A		     =					tokenizer.cls_token_id if cls_token_id is None else cls_token_id
     __A		     =					kwargs.pop('''sep_token_id''',			_lowerCamelCase    )
     __A		     =					tokenizer.sep_token_id if sep_token_id is None else sep_token_id
     __A		     =					kwargs.pop('''pad_token_id''',			_lowerCamelCase    )
     __A		     =					tokenizer.pad_token_id if pad_token_id is None else pad_token_id

     __A		     =					tokenizer.get_vocab()
     __A		     =					sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()],			key=lambda _lowerCamelCase    : x[1]    )
     __A		     =					[entry[0] for entry in vocab]
     return cls(
         vocab_list=_lowerCamelCase,			do_lower_case=_lowerCamelCase,			cls_token_id=_lowerCamelCase,			sep_token_id=_lowerCamelCase,			pad_token_id=_lowerCamelCase,			**_lowerCamelCase,			)


   @classmethod
   def 							_SCREAMING_SNAKE_CASE							(      cls					:	Optional[Any],			_lowerCamelCase					:	Union[str, os.PathLike],			*_lowerCamelCase					:	Any,			**_lowerCamelCase					:	List[str]    ):

     '''simple docstring'''
     try:
       __A		     =					BertTokenizer.from_pretrained(_lowerCamelCase,			*_lowerCamelCase,			**_lowerCamelCase    )
     except:  # noqa: E722
       from .tokenization_bert_fast import BertTokenizerFast

       __A		     =					BertTokenizerFast.from_pretrained(_lowerCamelCase,			*_lowerCamelCase,			**_lowerCamelCase    )
     return cls.from_tokenizer(_lowerCamelCase,			**_lowerCamelCase    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	List[Any],			_lowerCamelCase					:	Tuple    ):

     '''simple docstring'''
     if self.do_lower_case:
       __A		     =					case_fold_utfa(_lowerCamelCase    )
     __A		     =					self.tf_tokenizer.tokenize(_lowerCamelCase    )
     return tokens.merge_dims(1,			-1    )


   def 							_SCREAMING_SNAKE_CASE							(      self					:	Dict,			_lowerCamelCase					:	Any,			_lowerCamelCase					:	Dict=None,			_lowerCamelCase					:	Optional[int]=None,			_lowerCamelCase					:	Union[str, Any]=None,			_lowerCamelCase					:	Any=None,			_lowerCamelCase					:	List[Any]=None,			_lowerCamelCase					:	str=None,			_lowerCamelCase					:	Dict=None,			):

     '''simple docstring'''
     if padding is None:
       __A		     =					self.padding
     if padding not in ("longest", "max_length"):
       raise ValueError('''Padding must be either \'longest\' or \'max_length\'!'''    )
     if max_length is not None and text_pair is not None:
       # Because we have to instantiate a Trimmer to do it properly
       raise ValueError('''max_length cannot be overridden at call time when truncating paired texts!'''    )
     if max_length is None:
       __A		     =					self.max_length
     if truncation is None:
       __A		     =					self.truncation
     if pad_to_multiple_of is None:
       __A		     =					self.pad_to_multiple_of
     if return_token_type_ids is None:
       __A		     =					self.return_token_type_ids
     if return_attention_mask is None:
       __A		     =					self.return_attention_mask
     if not isinstance(_lowerCamelCase,			tf.Tensor    ):
       __A		     =					tf.convert_to_tensor(_lowerCamelCase    )
     if text_pair is not None and not isinstance(_lowerCamelCase,			tf.Tensor    ):
       __A		     =					tf.convert_to_tensor(_lowerCamelCase    )
     if text_pair is not None:
       if text.shape.rank > 1:
         raise ValueError('''text argument should not be multidimensional when a text pair is supplied!'''    )
       if text_pair.shape.rank > 1:
         raise ValueError('''text_pair should not be multidimensional!'''    )
     if text.shape.rank == 2:
       __A     ,       __A		     =					text[:, 0], text[:, 1]
     __A		     =					self.unpaired_tokenize(_lowerCamelCase    )
     if text_pair is None:  # Unpaired text
       if truncation:
         __A		     =					text[:, : max_length - 2]  # Allow room for special tokens
       __A     ,       __A		     =					combine_segments(
           (text,),			start_of_sequence_id=self.cls_token_id,			end_of_segment_id=self.sep_token_id    )
     else:  # Paired text
       __A		     =					self.unpaired_tokenize(_lowerCamelCase    )
       if truncation:
         __A     ,       __A		     =					self.paired_trimmer.trim([text, text_pair]    )
       __A     ,       __A		     =					combine_segments(
           (text, text_pair),			start_of_sequence_id=self.cls_token_id,			end_of_segment_id=self.sep_token_id    )
     if padding == "longest":
       __A		     =					input_ids.bounding_shape(axis=1    )
       if pad_to_multiple_of is not None:
         # No ceiling division in tensorflow, so we negate floordiv instead
         __A		     =					pad_to_multiple_of * (-tf.math.floordiv(-pad_length,			_lowerCamelCase    ))
     else:
       __A		     =					max_length

     __A     ,       __A		     =					pad_model_inputs(_lowerCamelCase,			max_seq_length=_lowerCamelCase,			pad_value=self.pad_token_id    )
     __A		     =					{'''input_ids''': input_ids}
     if return_attention_mask:
       __A		     =					attention_mask
     if return_token_type_ids:
       __A     ,       __A		     =					pad_model_inputs(
           _lowerCamelCase,			max_seq_length=_lowerCamelCase,			pad_value=self.pad_token_id    )
       __A		     =					token_type_ids
     return output






   def 							_SCREAMING_SNAKE_CASE							(      self					:	Optional[Any]    ):

     '''simple docstring'''
     return {
         "vocab_list": self.vocab_list,
         "do_lower_case": self.do_lower_case,
         "cls_token_id": self.cls_token_id,
         "sep_token_id": self.sep_token_id,
         "pad_token_id": self.pad_token_id,
     }




