



import json
import os
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation

A_		:  Union[str, Any]										=  logging.get_logger(__name__)

A_		:  Dict										=  {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

A_		:  Optional[Any]										=  {
    '''vocab_file''': {
        '''microsoft/deberta-base''': '''https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json''',
        '''microsoft/deberta-large''': '''https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json''',
        '''microsoft/deberta-xlarge''': '''https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json''',
        '''microsoft/deberta-base-mnli''': '''https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json''',
        '''microsoft/deberta-large-mnli''': '''https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json''',
        '''microsoft/deberta-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''microsoft/deberta-base''': '''https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt''',
        '''microsoft/deberta-large''': '''https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt''',
        '''microsoft/deberta-xlarge''': '''https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt''',
        '''microsoft/deberta-base-mnli''': '''https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt''',
        '''microsoft/deberta-large-mnli''': '''https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt''',
        '''microsoft/deberta-xlarge-mnli''': (
            '''https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'''
        ),
    },
}

A_		:  int										=  {
    '''microsoft/deberta-base''': 5_12,
    '''microsoft/deberta-large''': 5_12,
    '''microsoft/deberta-xlarge''': 5_12,
    '''microsoft/deberta-base-mnli''': 5_12,
    '''microsoft/deberta-large-mnli''': 5_12,
    '''microsoft/deberta-xlarge-mnli''': 5_12,
}

A_		:  List[str]										=  {
    '''microsoft/deberta-base''': {'''do_lower_case''': False},
    '''microsoft/deberta-large''': {'''do_lower_case''': False},
}


def 						UpperCAmelCase__   (   ):






 '''simple docstring'''

 a							=		(
     list(range(ord("!"       )			,       ord("~"       ) + 1       )       ) + list(range(ord("¡"       )			,       ord("¬"       ) + 1       )       ) + list(range(ord("®"       )			,       ord("ÿ"       ) + 1       )       )
 )
 a							=		bs[:]
 a							=		0
 for b in range(2**8       ):
  if b not in bs:
   bs.append(UpperCAmelCase__       )
   cs.append(2**8 + n       )
   n += 1
 a							=		[chr(UpperCAmelCase__       ) for n in cs]
 return dict(zip(UpperCAmelCase__			,       UpperCAmelCase__       )       )


def 						UpperCAmelCase__   (   UpperCAmelCase__			:Optional[int]       ):






 '''simple docstring'''

 a							=		set()
 a							=		word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)       )
  a							=		char
 return pairs




class 		_lowercase		(							UpperCAmelCase__						):
   _UpperCAmelCase  						=      VOCAB_FILES_NAMES
   _UpperCAmelCase  						=      PRETRAINED_VOCAB_FILES_MAP
   _UpperCAmelCase  						=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _UpperCAmelCase  						=      ['''input_ids''', '''attention_mask''', '''token_type_ids''']

   def __init__(   self      :							str				, __lowerCAmelCase      :							List[Any]				, __lowerCAmelCase      :							int				, __lowerCAmelCase      :							List[Any]="replace"				, __lowerCAmelCase      :							int="[CLS]"				, __lowerCAmelCase      :							List[str]="[SEP]"				, __lowerCAmelCase      :							Any="[SEP]"				, __lowerCAmelCase      :							Optional[Any]="[CLS]"				, __lowerCAmelCase      :							Optional[Any]="[UNK]"				, __lowerCAmelCase      :							Tuple="[PAD]"				, __lowerCAmelCase      :							List[str]="[MASK]"				, __lowerCAmelCase      :							Optional[Any]=False				, __lowerCAmelCase      :							Optional[int]=False				, **__lowerCAmelCase      :							Any				, )							->					Dict:


    """simple docstring"""






    a							=		AddedToken(__lowerCAmelCase				, lstrip=__lowerCAmelCase				, rstrip=__lowerCAmelCase				) if isinstance(__lowerCAmelCase				, __lowerCAmelCase				) else bos_token
    a							=		AddedToken(__lowerCAmelCase				, lstrip=__lowerCAmelCase				, rstrip=__lowerCAmelCase				) if isinstance(__lowerCAmelCase				, __lowerCAmelCase				) else eos_token
    a							=		AddedToken(__lowerCAmelCase				, lstrip=__lowerCAmelCase				, rstrip=__lowerCAmelCase				) if isinstance(__lowerCAmelCase				, __lowerCAmelCase				) else sep_token
    a							=		AddedToken(__lowerCAmelCase				, lstrip=__lowerCAmelCase				, rstrip=__lowerCAmelCase				) if isinstance(__lowerCAmelCase				, __lowerCAmelCase				) else cls_token
    a							=		AddedToken(__lowerCAmelCase				, lstrip=__lowerCAmelCase				, rstrip=__lowerCAmelCase				) if isinstance(__lowerCAmelCase				, __lowerCAmelCase				) else unk_token
    a							=		AddedToken(__lowerCAmelCase				, lstrip=__lowerCAmelCase				, rstrip=__lowerCAmelCase				) if isinstance(__lowerCAmelCase				, __lowerCAmelCase				) else pad_token

    # Mask token behave like a normal word, i.e. include the space before it
    a							=		AddedToken(__lowerCAmelCase				, lstrip=__lowerCAmelCase				, rstrip=__lowerCAmelCase				) if isinstance(__lowerCAmelCase				, __lowerCAmelCase				) else mask_token

    super().__init__(
        errors=__lowerCAmelCase				, bos_token=__lowerCAmelCase				, eos_token=__lowerCAmelCase				, unk_token=__lowerCAmelCase				, sep_token=__lowerCAmelCase				, cls_token=__lowerCAmelCase				, pad_token=__lowerCAmelCase				, mask_token=__lowerCAmelCase				, add_prefix_space=__lowerCAmelCase				, add_bos_token=__lowerCAmelCase				, **__lowerCAmelCase				, )
    a							=		add_bos_token

    with open(__lowerCAmelCase				, encoding="utf-8"				) as vocab_handle:
     a							=		json.load(__lowerCAmelCase				)
    a							=		{v: k for k, v in self.encoder.items()}
    a							=		errors  # how to handle errors in decoding
    a							=		bytes_to_unicode()
    a							=		{v: k for k, v in self.byte_encoder.items()}
    with open(__lowerCAmelCase				, encoding="utf-8"				) as merges_handle:
     a							=		merges_handle.read().split("\n"				)[1:-1]
    a							=		[tuple(merge.split()				) for merge in bpe_merges]
    a							=		dict(zip(__lowerCAmelCase				, range(len(__lowerCAmelCase				)				)				)				)
    a							=		{}
    a							=		add_prefix_space

    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    a							=		re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"				)

   @property
   # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.vocab_size
   def 		A				(   self      :							List[Any]				)							->					int:


    """simple docstring"""






    return len(self.encoder				)

   def 		A				(   self      :							Dict				)							->					str:


    """simple docstring"""






    return dict(self.encoder				, **self.added_tokens_encoder				)

   def 		A				(   self      :							Any				, __lowerCAmelCase      :							Optional[int]				)							->					List[Any]:


    """simple docstring"""






    if token in self.cache:
     return self.cache[token]
    a							=		tuple(__lowerCAmelCase				)
    a							=		get_pairs(__lowerCAmelCase				)

    if not pairs:
     return token

    while True:
     a							=		min(__lowerCAmelCase				, key=lambda __lowerCAmelCase				: self.bpe_ranks.get(__lowerCAmelCase				, float("inf"				)				)				)
     if bigram not in self.bpe_ranks:
      break
     a		,	a							=		bigram
     a							=		[]
     a							=		0
     while i < len(__lowerCAmelCase				):
      try:
       a							=		word.index(__lowerCAmelCase				, __lowerCAmelCase				)
      except ValueError:
       new_word.extend(word[i:]				)
       break
      else:
       new_word.extend(word[i:j]				)
       a							=		j

      if word[i] == first and i < len(__lowerCAmelCase				) - 1 and word[i + 1] == second:
       new_word.append(first + second				)
       i += 2
      else:
       new_word.append(word[i]				)
       i += 1
     a							=		tuple(__lowerCAmelCase				)
     a							=		new_word
     if len(__lowerCAmelCase				) == 1:
      break
     else:
      a							=		get_pairs(__lowerCAmelCase				)
    a							=		" ".join(__lowerCAmelCase				)
    a							=		word
    return word

   def 		A				(   self      :							Union[str, Any]				, __lowerCAmelCase      :							List[int]				, __lowerCAmelCase      :							Optional[List[int]] = None				)							->					List[int]:


    """simple docstring"""






    if token_ids_a is None:
     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    a							=		[self.cls_token_id]
    a							=		[self.sep_token_id]
    return cls + token_ids_a + sep + token_ids_a + sep

   def 		A				(   self      :							Optional[Any]				, __lowerCAmelCase      :							List[int]				, __lowerCAmelCase      :							Optional[List[int]] = None				, __lowerCAmelCase      :							bool = False				)							->					List[int]:


    """simple docstring"""






    if already_has_special_tokens:
     return super().get_special_tokens_mask(
         token_ids_a=__lowerCAmelCase				, token_ids_a=__lowerCAmelCase				, already_has_special_tokens=__lowerCAmelCase				)

    if token_ids_a is None:
     return [1] + ([0] * len(__lowerCAmelCase				)) + [1]
    return [1] + ([0] * len(__lowerCAmelCase				)) + [1] + ([0] * len(__lowerCAmelCase				)) + [1]

   def 		A				(   self      :							Union[str, Any]				, __lowerCAmelCase      :							List[int]				, __lowerCAmelCase      :							Optional[List[int]] = None				)							->					List[int]:


    """simple docstring"""






    a							=		[self.sep_token_id]
    a							=		[self.cls_token_id]

    if token_ids_a is None:
     return len(cls + token_ids_a + sep				) * [0]
    return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]

   def 		A				(   self      :							List[str]				, __lowerCAmelCase      :							Tuple				)							->					List[Any]:


    """simple docstring"""






    a							=		[]
    for token in re.findall(self.pat				, __lowerCAmelCase				):
     a							=		"".join(
         self.byte_encoder[b] for b in token.encode("utf-8"				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
     bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCAmelCase				).split(" "				)				)
    return bpe_tokens

   def 		A				(   self      :							Optional[int]				, __lowerCAmelCase      :							Optional[int]				)							->					Optional[int]:


    """simple docstring"""






    return self.encoder.get(__lowerCAmelCase				, self.encoder.get(self.unk_token				)				)

   def 		A				(   self      :							Tuple				, __lowerCAmelCase      :							Union[str, Any]				)							->					Union[str, Any]:


    """simple docstring"""






    return self.decoder.get(__lowerCAmelCase				)

   def 		A				(   self      :							List[str]				, __lowerCAmelCase      :							List[Any]				)							->					str:


    """simple docstring"""






    a							=		"".join(__lowerCAmelCase				)
    a							=		bytearray([self.byte_decoder[c] for c in text]				).decode("utf-8"				, errors=self.errors				)
    return text

   def 		A				(   self      :							Dict				, __lowerCAmelCase      :							str				, __lowerCAmelCase      :							Optional[str] = None				)							->					Tuple[str]:


    """simple docstring"""






    if not os.path.isdir(__lowerCAmelCase				):
     logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
     return
    a							=		os.path.join(
        __lowerCAmelCase				, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]				)
    a							=		os.path.join(
        __lowerCAmelCase				, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]				)

    with open(__lowerCAmelCase				, "w"				, encoding="utf-8"				) as f:
     f.write(json.dumps(self.encoder				, indent=2				, sort_keys=__lowerCAmelCase				, ensure_ascii=__lowerCAmelCase				) + "\n"				)

    a							=		0
    with open(__lowerCAmelCase				, "w"				, encoding="utf-8"				) as writer:
     writer.write("#version: 0.2\n"				)
     for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				, key=lambda __lowerCAmelCase				: kv[1]				):
      if index != token_index:
       logger.warning(
           f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
           " Please check that the tokenizer is not corrupted!"				)
       a							=		token_index
      writer.write(" ".join(__lowerCAmelCase				) + "\n"				)
      index += 1

    return vocab_file, merge_file

   def 		A				(   self      :							Union[str, Any]				, __lowerCAmelCase      :							Union[str, Any]				, __lowerCAmelCase      :							List[Any]=False				, **__lowerCAmelCase      :							Union[str, Any]				)							->					Union[str, Any]:


    """simple docstring"""






    a							=		kwargs.pop("add_prefix_space"				, self.add_prefix_space				)
    if (is_split_into_words or add_prefix_space) and (len(__lowerCAmelCase				) > 0 and not text[0].isspace()):
     a							=		" " + text
    return (text, kwargs)




   def 		A				(   self      :							Dict				, __lowerCAmelCase      :							"Conversation"				)							->					List[int]:


    """simple docstring"""






    a							=		[]
    for is_user, text in conversation.iter_texts():
     input_ids.extend(self.encode(__lowerCAmelCase				, add_special_tokens=__lowerCAmelCase				) + [self.eos_token_id]				)
    if len(__lowerCAmelCase				) > self.model_max_length:
     a							=		input_ids[-self.model_max_length :]
    return input_ids








import argparse
import os


UpperCAmelCase__    :      Any       					={
    '''pytorch''': {
        '''1.13''': {
            '''torch''': '''1.13.1''',
            '''torchvision''': '''0.14.1''',
            '''torchaudio''': '''0.13.1''',
            '''python''': 3.9,
            '''cuda''': '''cu116''',
            '''install''': (
                '''python3 -m pip install --no-cache-dir -U torch==1.13.1 torchvision==0.14.1 torchaudio==0.13.1'''
                ''' --extra-index-url https://download.pytorch.org/whl/cu116'''
            ),
            '''base_image''': '''nvidia/cuda:11.6.2-cudnn8-devel-ubuntu20.04''',
        },
        '''1.12''': {
            '''torch''': '''1.12.1''',
            '''torchvision''': '''0.13.1''',
            '''torchaudio''': '''0.12.1''',
            '''python''': 3.9,
            '''cuda''': '''cu113''',
            '''install''': (
                '''python3 -m pip install --no-cache-dir -U torch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1'''
                ''' --extra-index-url https://download.pytorch.org/whl/cu113'''
            ),
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''1.11''': {
            '''torch''': '''1.11.0''',
            '''torchvision''': '''0.12.0''',
            '''torchaudio''': '''0.11.0''',
            '''python''': 3.9,
            '''cuda''': '''cu113''',
            '''install''': (
                '''python3 -m pip install --no-cache-dir -U torch==1.11.0 torchvision==0.12.0 torchaudio==0.11.0'''
                ''' --extra-index-url https://download.pytorch.org/whl/cu113'''
            ),
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''1.10''': {
            '''torch''': '''1.10.2''',
            '''torchvision''': '''0.11.3''',
            '''torchaudio''': '''0.10.2''',
            '''python''': 3.9,
            '''cuda''': '''cu113''',
            '''install''': (
                '''python3 -m pip install --no-cache-dir -U torch==1.10.2 torchvision==0.11.3 torchaudio==0.10.2'''
                ''' --extra-index-url https://download.pytorch.org/whl/cu113'''
            ),
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        # torchaudio < 0.10 has no CUDA-enabled binary distributions
        '''1.9''': {
            '''torch''': '''1.9.1''',
            '''torchvision''': '''0.10.1''',
            '''torchaudio''': '''0.9.1''',
            '''python''': 3.9,
            '''cuda''': '''cu111''',
            '''install''': (
                '''python3 -m pip install --no-cache-dir -U torch==1.9.1 torchvision==0.10.1 torchaudio==0.9.1'''
                ''' --extra-index-url https://download.pytorch.org/whl/cu111'''
            ),
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
    },
    '''tensorflow''': {
        '''2.11''': {
            '''tensorflow''': '''2.11.1''',
            '''install''': '''python3 -m pip install --no-cache-dir -U tensorflow==2.11.1''',
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''2.10''': {
            '''tensorflow''': '''2.10.1''',
            '''install''': '''python3 -m pip install --no-cache-dir -U tensorflow==2.10.1''',
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''2.9''': {
            '''tensorflow''': '''2.9.3''',
            '''install''': '''python3 -m pip install --no-cache-dir -U tensorflow==2.9.3''',
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''2.8''': {
            '''tensorflow''': '''2.8.2''',
            '''install''': '''python3 -m pip install --no-cache-dir -U tensorflow==2.8.2''',
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''2.7''': {
            '''tensorflow''': '''2.7.3''',
            '''install''': '''python3 -m pip install --no-cache-dir -U tensorflow==2.7.3''',
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''2.6''': {
            '''tensorflow''': '''2.6.5''',
            '''install''': '''python3 -m pip install --no-cache-dir -U tensorflow==2.6.5''',
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
        '''2.5''': {
            '''tensorflow''': '''2.5.3''',
            '''install''': '''python3 -m pip install --no-cache-dir -U tensorflow==2.5.3''',
            '''base_image''': '''nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04''',
        },
    },
}


if __name__ == "__main__":
							UpperCAmelCase__    :      Tuple       					=argparse.ArgumentParser('''Choose the framework and version to install''')
							parser.add_argument(
							    '''--framework''', help='''The framework to install. Should be `torch` or `tensorflow`''', type=str, required=True
							)
							parser.add_argument('''--version''', help='''The version of the framework to install.''', type=str, required=True)
							UpperCAmelCase__    :      List[str]       					=parser.parse_args()

							UpperCAmelCase__    :      Dict       					=past_versions_testing[args.framework][args.version]

							os.system(F"echo \"export INSTALL_CMD=\'{info['install']}\'\" >> ~/.profile")
							print(F"echo \"export INSTALL_CMD=\'{info['install']}\'\" >> ~/.profile")

							UpperCAmelCase__    :      List[Any]       					=''''''
							if args.framework == "pytorch":
														UpperCAmelCase__    :      Optional[int]       					=info['''cuda''']
							os.system(F"echo \"export CUDA='{cuda}'\" >> ~/.profile")
							print(F"echo \"export CUDA='{cuda}'\" >> ~/.profile")





