



'''simple docstring'''






import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin



class 							lowercase       (  _lowerCamelCase					,					unittest.TestCase				):


 """simple docstring"""


 UpperCAmelCase							=  DebertaTokenizer
 UpperCAmelCase							=  True
 UpperCAmelCase							=  DebertaTokenizerFast







 def _snake_case (							self							)		->   Any:
        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        _UpperCAmelCase  :						int			     = [
            """l""",
            """o""",
            """w""",
            """e""",
            """r""",
            """s""",
            """t""",
            """i""",
            """d""",
            """n""",
            """\u0120""",
            """\u0120l""",
            """\u0120n""",
            """\u0120lo""",
            """\u0120low""",
            """er""",
            """\u0120lowest""",
            """\u0120newer""",
            """\u0120wider""",
            """[UNK]""",
        ]
        _UpperCAmelCase  :						Optional[int]			     = dict(zip(a_  ,range(len(a_							)							)							)							)
        _UpperCAmelCase  :						Tuple			     = ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
        _UpperCAmelCase  :						Any			     = {"""unk_token""": """[UNK]"""}

        _UpperCAmelCase  :						Dict			     = os.path.join(self.tmpdirname  ,VOCAB_FILES_NAMES["""vocab_file"""]							)
        _UpperCAmelCase  :						List[Any]			     = os.path.join(self.tmpdirname  ,VOCAB_FILES_NAMES["""merges_file"""]							)
        with open(self.vocab_file  ,"""w"""  ,encoding="""utf-8"""							) as fp:
               fp.write(json.dumps(a_							) + """\n"""							)
        with open(self.merges_file  ,"""w"""  ,encoding="""utf-8"""							) as fp:
               fp.write("""\n""".join(a_							)							)







 def _snake_case (							self  ,**a_							)		->   Optional[Any]:
        kwargs.update(self.special_tokens_map							)
        return self.tokenizer_class.from_pretrained(self.tmpdirname  ,**a_							)







 def _snake_case (							self  ,a_							)		->   Union[str, Any]:
        _UpperCAmelCase  :						Tuple			     = """lower newer"""
        _UpperCAmelCase  :						Tuple			     = """lower newer"""
        return input_text, output_text







 def _snake_case (							self							)		->   Optional[Any]:
        _UpperCAmelCase  :						Optional[Any]			     = self.get_tokenizer()
        _UpperCAmelCase  :						Dict			     = """lower newer"""
        _UpperCAmelCase  :						Optional[int]			     = ["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
        _UpperCAmelCase  :						Optional[int]			     = tokenizer.tokenize(a_							)
        self.assertListEqual(a_  ,a_							)

        _UpperCAmelCase  :						int			     = tokens + [tokenizer.unk_token]
        _UpperCAmelCase  :						Optional[Any]			     = [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(a_							)  ,a_							)







 def _snake_case (							self							)		->   Optional[Any]:
        _UpperCAmelCase  :						Union[str, Any]			     = self.get_tokenizer()
        _UpperCAmelCase  :						Union[str, Any]			     = tokenizer("""Hello"""  ,"""World"""							)
        _UpperCAmelCase  :						Dict			     = [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
        self.assertListEqual(tokd["""token_type_ids"""]  ,a_							)







 @slow
 def _snake_case (							self							)		->   Optional[Any]:
        _UpperCAmelCase  :						List[str]			     = self.tokenizer_class.from_pretrained("""microsoft/deberta-base"""							)

        _UpperCAmelCase  :						List[Any]			     = tokenizer.encode("""sequence builders"""  ,add_special_tokens=a_							)
        _UpperCAmelCase  :						List[Any]			     = tokenizer.encode("""multi-sequence build"""  ,add_special_tokens=a_							)

        _UpperCAmelCase  :						Tuple			     = tokenizer.encode(
            """sequence builders"""  ,add_special_tokens=a_  ,add_prefix_space=a_							)
        _UpperCAmelCase  :						str			     = tokenizer.encode(
            """sequence builders"""  ,"""multi-sequence build"""  ,add_special_tokens=a_  ,add_prefix_space=a_							)

        _UpperCAmelCase  :						Optional[int]			     = tokenizer.build_inputs_with_special_tokens(a_							)
        _UpperCAmelCase  :						List[str]			     = tokenizer.build_inputs_with_special_tokens(a_  ,a_							)

        assert encoded_sentence == encoded_text_from_decode
        assert encoded_pair == encoded_pair_from_decode







 @slow
 def _snake_case (							self							)		->   List[Any]:
        _UpperCAmelCase  :						Union[str, Any]			     = [self.tokenizer_class]
        if self.test_rust_tokenizer:
               tokenizer_classes.append(self.rust_tokenizer_class							)

        for tokenizer_class in tokenizer_classes:
               _UpperCAmelCase  :						Union[str, Any]			     = tokenizer_class.from_pretrained("""microsoft/deberta-base"""							)

               _UpperCAmelCase  :						List[str]			     = [
                   """ALBERT: A Lite BERT for Self-supervised Learning of Language Representations""",
                   """ALBERT incorporates two parameter reduction techniques""",
                   """The first one is a factorized embedding parameterization. By decomposing the large vocabulary"""
                   """ embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"""
                   """ vocabulary embedding.""",
               ]

               _UpperCAmelCase  :						List[str]			     = tokenizer(a_  ,padding=a_							)
               _UpperCAmelCase  :						Optional[int]			     = [tokenizer.decode(a_  ,skip_special_tokens=a_							) for seq in encoding["""input_ids"""]]

               # fmt: off
               _UpperCAmelCase  :						Any			     = {
                   """input_ids""": [
                       [1, 2_118, 11_126, 565, 35, 83, 25_191, 163, 18_854, 13, 12_156, 12, 16_101, 25_376, 13_807, 9, 22_205, 27_893, 1_635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                       [1, 2_118, 11_126, 565, 24_536, 80, 43_797, 4_878, 7_373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                       [1, 133, 78, 65, 16, 10, 3_724, 1_538, 33_183, 11_303, 43_797, 1_938, 4, 870, 24_165, 29_105, 5, 739, 32_644, 33_183, 11_303, 36_173, 88, 80, 650, 7_821, 45_940, 6, 52, 2_559, 5, 1_836, 9, 5, 7_397, 13_171, 31, 5, 1_836, 9, 32_644, 33_183, 11_303, 4, 2]
                   ],
                   """token_type_ids""": [
                       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                   ],
                   """attention_mask""": [
                       [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                       [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                       [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
                   ]
               }
               # fmt: on

               _UpperCAmelCase  :						Optional[Any]			     = [
                   """ALBERT: A Lite BERT for Self-supervised Learning of Language Representations""",
                   """ALBERT incorporates two parameter reduction techniques""",
                   """The first one is a factorized embedding parameterization. By decomposing the large vocabulary"""
                   """ embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"""
                   """ vocabulary embedding.""",
               ]

               self.assertDictEqual(encoding.data  ,a_							)

               for expected, decoded in zip(a_  ,a_							):
                      self.assertEqual(a_  ,a_							)








'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


A_							: Optional[Any]						 =	{
    """configuration_llama""": ["""LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP""", """LlamaConfig"""],
}

try:
   if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_							: List[str]						 =	["""LlamaTokenizer"""]

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_							: Dict						 =	["""LlamaTokenizerFast"""]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_							: Optional[int]						 =	[
       """LlamaForCausalLM""",
       """LlamaModel""",
       """LlamaPreTrainedModel""",
       """LlamaForSequenceClassification""",
   ]


if TYPE_CHECKING:
   from .configuration_llama import LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP, LlamaConfig

   try:
      if not is_sentencepiece_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_llama import LlamaTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_llama_fast import LlamaTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_llama import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel, LlamaPreTrainedModel


else:
   import sys

   A_							: Any						 =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



