

import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin





class  __lowerCAmelCase     (					SCREAMING_SNAKE_CASE			,			unittest.TestCase							):
  _a									=  DebertaTokenizer
  _a									=  True
  _a									=  DebertaTokenizerFast




  def   A__     (				self					)  -> Optional[Any]:
        '''simple docstring'''






        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        _lowercase	       =[
            'l',
            'o',
            'w',
            'e',
            'r',
            's',
            't',
            'i',
            'd',
            'n',
            '\u0120',
            '\u0120l',
            '\u0120n',
            '\u0120lo',
            '\u0120low',
            'er',
            '\u0120lowest',
            '\u0120newer',
            '\u0120wider',
            '[UNK]',
        ]
        _lowercase	       =dict(zip(lowerCAmelCase							,   range(len(lowerCAmelCase					)					)					)					)
        _lowercase	       =['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
        _lowercase	       ={'unk_token': '[UNK]'}

        _lowercase	       =os.path.join(self.tmpdirname							,   VOCAB_FILES_NAMES['vocab_file']					)
        _lowercase	       =os.path.join(self.tmpdirname							,   VOCAB_FILES_NAMES['merges_file']					)
        with open(self.vocab_file							,   'w'							,   encoding='utf-8'					) as fp:
              fp.write(json.dumps(lowerCAmelCase					) + '\n'					)
        with open(self.merges_file							,   'w'							,   encoding='utf-8'					) as fp:
              fp.write('\n'.join(lowerCAmelCase					)					)




  def   A__     (				self							,   **lowerCAmelCase					)  -> str:
        '''simple docstring'''






        kwargs.update(self.special_tokens_map					)
        return self.tokenizer_class.from_pretrained(self.tmpdirname							,   **lowerCAmelCase					)




  def   A__     (				self							,   lowerCAmelCase					)  -> Any:
        '''simple docstring'''






        _lowercase	       ='lower newer'
        _lowercase	       ='lower newer'
        return input_text, output_text




  def   A__     (				self					)  -> List[str]:
        '''simple docstring'''






        _lowercase	       =self.get_tokenizer()
        _lowercase	       ='lower newer'
        _lowercase	       =['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
        _lowercase	       =tokenizer.tokenize(lowerCAmelCase					)
        self.assertListEqual(lowerCAmelCase							,   lowerCAmelCase					)

        _lowercase	       =tokens + [tokenizer.unk_token]
        _lowercase	       =[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase					)							,   lowerCAmelCase					)




  def   A__     (				self					)  -> Dict:
        '''simple docstring'''






        _lowercase	       =self.get_tokenizer()
        _lowercase	       =tokenizer('Hello'							,   'World'					)
        _lowercase	       =[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
        self.assertListEqual(tokd['token_type_ids']							,   lowerCAmelCase					)




  @slow
  def   A__     (				self					)  -> Any:
        '''simple docstring'''






        _lowercase	       =self.tokenizer_class.from_pretrained('microsoft/deberta-base'					)

        _lowercase	       =tokenizer.encode('sequence builders'							,   add_special_tokens=lowerCAmelCase					)
        _lowercase	       =tokenizer.encode('multi-sequence build'							,   add_special_tokens=lowerCAmelCase					)

        _lowercase	       =tokenizer.encode(
            'sequence builders'							,   add_special_tokens=lowerCAmelCase							,   add_prefix_space=lowerCAmelCase					)
        _lowercase	       =tokenizer.encode(
            'sequence builders'							,   'multi-sequence build'							,   add_special_tokens=lowerCAmelCase							,   add_prefix_space=lowerCAmelCase					)

        _lowercase	       =tokenizer.build_inputs_with_special_tokens(lowerCAmelCase					)
        _lowercase	       =tokenizer.build_inputs_with_special_tokens(lowerCAmelCase							,   lowerCAmelCase					)

        assert encoded_sentence == encoded_text_from_decode
        assert encoded_pair == encoded_pair_from_decode




  @slow
  def   A__     (				self					)  -> List[str]:
        '''simple docstring'''






        _lowercase	       =[self.tokenizer_class]
        if self.test_rust_tokenizer:
              tokenizer_classes.append(self.rust_tokenizer_class					)

        for tokenizer_class in tokenizer_classes:
              _lowercase	       =tokenizer_class.from_pretrained('microsoft/deberta-base'					)

              _lowercase	       =[
                  'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
                  'ALBERT incorporates two parameter reduction techniques',
                  'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
                  ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
                  ' vocabulary embedding.',
              ]

              _lowercase	       =tokenizer(lowerCAmelCase							,   padding=lowerCAmelCase					)
              _lowercase	       =[tokenizer.decode(lowerCAmelCase							,   skip_special_tokens=lowerCAmelCase					) for seq in encoding['input_ids']]

              # fmt: off
              _lowercase	       ={
                  'input_ids': [
                      [1, 2_118, 11_126, 565, 35, 83, 25_191, 163, 18_854, 13, 12_156, 12, 16_101, 25_376, 13_807, 9, 22_205, 27_893, 1_635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                      [1, 2_118, 11_126, 565, 24_536, 80, 43_797, 4_878, 7_373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                      [1, 133, 78, 65, 16, 10, 3_724, 1_538, 33_183, 11_303, 43_797, 1_938, 4, 870, 24_165, 29_105, 5, 739, 32_644, 33_183, 11_303, 36_173, 88, 80, 650, 7_821, 45_940, 6, 52, 2_559, 5, 1_836, 9, 5, 7_397, 13_171, 31, 5, 1_836, 9, 32_644, 33_183, 11_303, 4, 2]
                  ],
                  'token_type_ids': [
                      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                  ],
                  'attention_mask': [
                      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
                  ]
              }
              # fmt: on

              _lowercase	       =[
                  'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
                  'ALBERT incorporates two parameter reduction techniques',
                  'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
                  ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
                  ' vocabulary embedding.',
              ]

              self.assertDictEqual(encoding.data							,   lowerCAmelCase					)

              for expected, decoded in zip(lowerCAmelCase							,   lowerCAmelCase					):
                    self.assertEqual(lowerCAmelCase							,   lowerCAmelCase					)





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase_									=   {'configuration_xlnet': ['XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP', 'XLNetConfig']}

try:
  if not is_sentencepiece_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase_									=   ['XLNetTokenizer']

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase_									=   ['XLNetTokenizerFast']

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase_									=   [
      'XLNET_PRETRAINED_MODEL_ARCHIVE_LIST',
      'XLNetForMultipleChoice',
      'XLNetForQuestionAnswering',
      'XLNetForQuestionAnsweringSimple',
      'XLNetForSequenceClassification',
      'XLNetForTokenClassification',
      'XLNetLMHeadModel',
      'XLNetModel',
      'XLNetPreTrainedModel',
      'load_tf_weights_in_xlnet',
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase_									=   [
      'TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST',
      'TFXLNetForMultipleChoice',
      'TFXLNetForQuestionAnsweringSimple',
      'TFXLNetForSequenceClassification',
      'TFXLNetForTokenClassification',
      'TFXLNetLMHeadModel',
      'TFXLNetMainLayer',
      'TFXLNetModel',
      'TFXLNetPreTrainedModel',
  ]


if TYPE_CHECKING:
  from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, XLNetConfig

  try:
    if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_xlnet import XLNetTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_xlnet_fast import XLNetTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_xlnet import (
        XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
        XLNetForMultipleChoice,
        XLNetForQuestionAnswering,
        XLNetForQuestionAnsweringSimple,
        XLNetForSequenceClassification,
        XLNetForTokenClassification,
        XLNetLMHeadModel,
        XLNetModel,
        XLNetPreTrainedModel,
        load_tf_weights_in_xlnet,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_xlnet import (
        TF_XLNET_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLNetForMultipleChoice,
        TFXLNetForQuestionAnsweringSimple,
        TFXLNetForSequenceClassification,
        TFXLNetForTokenClassification,
        TFXLNetLMHeadModel,
        TFXLNetMainLayer,
        TFXLNetModel,
        TFXLNetPreTrainedModel,
    )

else:
  import sys

  lowercase_									=   _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


