




import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__       :      Union[str, Any]  =   get_tests_dir('fixtures/test_sentencepiece.model')




@require_sentencepiece
@require_tokenizers
class 		UpperCamelCase__	(lowerCAmelCase__    ,				unittest.TestCase     ):

    '''simple docstring'''



    lowerCamelCase_      :	Optional[Any]	     =      XLNetTokenizer
    lowerCamelCase_      :	Union[str, Any]	     =      XLNetTokenizerFast
    lowerCamelCase_      :	Optional[int]	     =      True
    lowerCamelCase_      :	Optional[int]	     =      True







    def     _lowercase     (    self     )     ->      str:
           super().setUp()

           # We have a SentencePiece fixture for testing
           lowerCamelCase   :      int			 =					XLNetTokenizer(UpperCamelCase__     ,  keep_accents=UpperCamelCase__     )
           tokenizer.sanitize_special_tokens()
           tokenizer.save_pretrained(self.tmpdirname     )







    def     _lowercase     (    self     )     ->      str:
           lowerCamelCase   :      str			 =					"<s>"
           lowerCamelCase   :      Any			 =					1

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase__     )     ,  UpperCamelCase__     )
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase__     )     ,  UpperCamelCase__     )







    def     _lowercase     (    self     )     ->      Any:
           lowerCamelCase   :      Union[str, Any]			 =					list(self.get_tokenizer().get_vocab().keys()     )

           self.assertEqual(vocab_keys[0]     ,  "<unk>"     )
           self.assertEqual(vocab_keys[1]     ,  "<s>"     )
           self.assertEqual(vocab_keys[-1]     ,  "<eod>"     )
           self.assertEqual(len(UpperCamelCase__     )     ,  1006     )







    def     _lowercase     (    self     )     ->      Optional[int]:
           self.assertEqual(self.get_tokenizer().vocab_size     ,  1000     )







    def     _lowercase     (    self     )     ->      List[Any]:
           lowerCamelCase   :      int			 =					XLNetTokenizer(UpperCamelCase__     ,  keep_accents=UpperCamelCase__     )

           lowerCamelCase   :      str			 =					tokenizer.tokenize("This is a test"     )
           self.assertListEqual(UpperCamelCase__     ,  ["▁This", "▁is", "▁a", "▁t", "est"]     )

           self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__     )     ,  [285, 46, 10, 170, 382]     )

           lowerCamelCase   :      int			 =					tokenizer.tokenize("I was born in 92000, and this is falsé."     )
           self.assertListEqual(
               UpperCamelCase__     ,  [
                   SPIECE_UNDERLINE + "I",
                   SPIECE_UNDERLINE + "was",
                   SPIECE_UNDERLINE + "b",
                   "or",
                   "n",
                   SPIECE_UNDERLINE + "in",
                   SPIECE_UNDERLINE + "",
                   "9",
                   "2",
                   "0",
                   "0",
                   "0",
                   ",",
                   SPIECE_UNDERLINE + "and",
                   SPIECE_UNDERLINE + "this",
                   SPIECE_UNDERLINE + "is",
                   SPIECE_UNDERLINE + "f",
                   "al",
                   "s",
                   "é",
                   ".",
               ]     ,  )
           lowerCamelCase   :      int			 =					tokenizer.convert_tokens_to_ids(UpperCamelCase__     )
           self.assertListEqual(UpperCamelCase__     ,  [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]     )

           lowerCamelCase   :      Dict			 =					tokenizer.convert_ids_to_tokens(UpperCamelCase__     )
           self.assertListEqual(
               UpperCamelCase__     ,  [
                   SPIECE_UNDERLINE + "I",
                   SPIECE_UNDERLINE + "was",
                   SPIECE_UNDERLINE + "b",
                   "or",
                   "n",
                   SPIECE_UNDERLINE + "in",
                   SPIECE_UNDERLINE + "",
                   "<unk>",
                   "2",
                   "0",
                   "0",
                   "0",
                   ",",
                   SPIECE_UNDERLINE + "and",
                   SPIECE_UNDERLINE + "this",
                   SPIECE_UNDERLINE + "is",
                   SPIECE_UNDERLINE + "f",
                   "al",
                   "s",
                   "<unk>",
                   ".",
               ]     ,  )







    def     _lowercase     (    self     )     ->      Any:
           lowerCamelCase   :      Union[str, Any]			 =					XLNetTokenizer(UpperCamelCase__     ,  do_lower_case=UpperCamelCase__     )
           lowerCamelCase   :      Dict			 =					tokenizer.tokenize("I was born in 92000, and this is falsé."     )
           self.assertListEqual(
               UpperCamelCase__     ,  [
                   SPIECE_UNDERLINE + "",
                   "i",
                   SPIECE_UNDERLINE + "was",
                   SPIECE_UNDERLINE + "b",
                   "or",
                   "n",
                   SPIECE_UNDERLINE + "in",
                   SPIECE_UNDERLINE + "",
                   "9",
                   "2",
                   "0",
                   "0",
                   "0",
                   ",",
                   SPIECE_UNDERLINE + "and",
                   SPIECE_UNDERLINE + "this",
                   SPIECE_UNDERLINE + "is",
                   SPIECE_UNDERLINE + "f",
                   "al",
                   "se",
                   ".",
               ]     ,  )
           self.assertListEqual(tokenizer.tokenize("H\u00E9llo"     )     ,  ["▁he", "ll", "o"]     )







    def     _lowercase     (    self     )     ->      int:
           lowerCamelCase   :      str			 =					XLNetTokenizer(UpperCamelCase__     ,  do_lower_case=UpperCamelCase__     )
           lowerCamelCase   :      Optional[int]			 =					tokenizer.tokenize("I was born in 92000, and this is falsé."     )
           self.assertListEqual(
               UpperCamelCase__     ,  [
                   SPIECE_UNDERLINE + "I",
                   SPIECE_UNDERLINE + "was",
                   SPIECE_UNDERLINE + "b",
                   "or",
                   "n",
                   SPIECE_UNDERLINE + "in",
                   SPIECE_UNDERLINE + "",
                   "9",
                   "2",
                   "0",
                   "0",
                   "0",
                   ",",
                   SPIECE_UNDERLINE + "and",
                   SPIECE_UNDERLINE + "this",
                   SPIECE_UNDERLINE + "is",
                   SPIECE_UNDERLINE + "f",
                   "al",
                   "se",
                   ".",
               ]     ,  )







    @slow
    def     _lowercase     (    self     )     ->      Tuple:
           lowerCamelCase   :      List[Any]			 =					XLNetTokenizer.from_pretrained("xlnet-base-cased"     )

           lowerCamelCase   :      List[str]			 =					tokenizer.encode("sequence builders"     ,  add_special_tokens=UpperCamelCase__     )
           lowerCamelCase   :      Dict			 =					tokenizer.encode("multi-sequence build"     ,  add_special_tokens=UpperCamelCase__     )

           lowerCamelCase   :      str			 =					tokenizer.build_inputs_with_special_tokens(UpperCamelCase__     )
           lowerCamelCase   :      List[str]			 =					tokenizer.build_inputs_with_special_tokens(UpperCamelCase__     ,  UpperCamelCase__     )

           assert encoded_sentence == text + [4, 3]
           assert encoded_pair == text + [4] + text_a + [4, 3]







    @slow
    def     _lowercase     (    self     )     ->      Union[str, Any]:
           # fmt: off
           lowerCamelCase   :      str			 =					{"input_ids": [[17, 2_1442, 270, 17, 10, 1_4645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 2_2018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 1_4431, 13, 5500, 11, 1176, 580, 13, 1_6819, 4797, 23, 17, 10, 1_7135, 658, 19, 457, 7932, 13, 184, 19, 3154, 1_7135, 6468, 19, 1404, 1_2269, 19, 4229, 5356, 1_6264, 46, 19, 17, 2_0545, 1_0395, 9, 9, 9, 11, 28, 6421, 9531, 2_0729, 17, 10, 353, 1_7022, 11, 21, 6421, 9531, 1_6949, 17, 10, 1_1509, 753, 11, 33, 95, 2421, 7385, 956, 1_4431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 2_4738, 19, 1_3203, 658, 218, 787, 21, 430, 1_8482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 2_2178, 27, 1064, 22, 956, 13, 1_1101, 1429, 5854, 2_4313, 1_8953, 40, 422, 2_4366, 68, 1758, 37, 1_0483, 1_4257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 1_3894, 3380, 23, 95, 18, 1_7634, 2288, 9, 4, 3]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=UpperCamelCase__     ,  model_name="xlnet-base-cased"     ,  revision="c841166438c31ec7ca9a106dee7bb312b73ae511"     ,  )











'''simple docstring'''






class 							lowercase       :


 """simple docstring"""


 def __init__(							self							)		->   List[str]:
        _UpperCAmelCase  :						int			     = 0
        _UpperCAmelCase  :						Union[str, Any]			     = 0
        _UpperCAmelCase  :						Optional[int]			     = {}







 def _snake_case (							self  ,a_							)		->   Optional[Any]:
        if vertex not in self.adjacency:
               _UpperCAmelCase  :						int			     = {}
               self.num_vertices += 1







 def _snake_case (							self  ,a_  ,a_  ,a_							)		->   int:
        self.add_vertex(a_							)
        self.add_vertex(a_							)

        if head == tail:
               return

        _UpperCAmelCase  :						List[Any]			     = weight
        _UpperCAmelCase  :						Dict			     = weight







 def _snake_case (							self							)		->   Dict:
        _UpperCAmelCase  :						Optional[int]			     = self.get_edges()
        for edge in edges:
               _UpperCAmelCase     ,_UpperCAmelCase     ,_UpperCAmelCase  :						Dict			     = edge
               edges.remove((tail, head, weight)							)
        for i in range(len(a_							)							):
               _UpperCAmelCase  :						str			     = list(edges[i]							)

        edges.sort(key=lambda a_							: e[2]							)
        for i in range(len(a_							) - 1							):
               if edges[i][2] >= edges[i + 1][2]:
                      _UpperCAmelCase  :						Optional[Any]			     = edges[i][2] + 1
        for edge in edges:
               _UpperCAmelCase     ,_UpperCAmelCase     ,_UpperCAmelCase  :						Union[str, Any]			     = edge
               _UpperCAmelCase  :						str			     = weight
               _UpperCAmelCase  :						List[str]			     = weight







 def __str__(							self							)		->   Any:
        _UpperCAmelCase  :						List[Any]			     = """"""
        for tail in self.adjacency:
               for head in self.adjacency[tail]:
                      _UpperCAmelCase  :						List[str]			     = self.adjacency[head][tail]
                      string += f'''{head} -> {tail} == {weight}\n'''
        return string.rstrip("""\n"""							)







 def _snake_case (							self							)		->   str:
        _UpperCAmelCase  :						int			     = []
        for tail in self.adjacency:
               for head in self.adjacency[tail]:
                      output.append((tail, head, self.adjacency[head][tail])							)
        return output







 def _snake_case (							self							)		->   Optional[int]:
        return self.adjacency.keys()







 @staticmethod
 def _snake_case (							a_=None  ,a_=None							)		->   Tuple:
        _UpperCAmelCase  :						List[Any]			     = Graph()
        if vertices is None:
               _UpperCAmelCase  :						List[str]			     = []
        if edges is None:
               _UpperCAmelCase  :						Optional[Any]			     = []
        for vertex in vertices:
               g.add_vertex(a_							)
        for edge in edges:
               g.add_edge(*a_							)
        return g



 class 							lowercase       :


  """simple docstring"""


  def __init__(							self							)		->   int:
         _UpperCAmelCase  :						List[str]			     = {}
         _UpperCAmelCase  :						int			     = {}







  def __len__(							self							)		->   Tuple:
         return len(self.parent							)







  def _snake_case (							self  ,a_							)		->   str:
         if item in self.parent:
                return self.find(a_							)

         _UpperCAmelCase  :						Optional[Any]			     = item
         _UpperCAmelCase  :						List[Any]			     = 0
         return item







  def _snake_case (							self  ,a_							)		->   List[str]:
         if item not in self.parent:
                return self.make_set(a_							)
         if item != self.parent[item]:
                _UpperCAmelCase  :						List[Any]			     = self.find(self.parent[item]							)
         return self.parent[item]







  def _snake_case (							self  ,a_  ,a_							)		->   Union[str, Any]:
         _UpperCAmelCase  :						Any			     = self.find(a_							)
         _UpperCAmelCase  :						List[str]			     = self.find(a_							)

         if roota == roota:
                return roota

         if self.rank[roota] > self.rank[roota]:
                _UpperCAmelCase  :						Any			     = roota
                return roota

         if self.rank[roota] < self.rank[roota]:
                _UpperCAmelCase  :						Any			     = roota
                return roota

         if self.rank[roota] == self.rank[roota]:
                self.rank[roota] += 1
                _UpperCAmelCase  :						List[str]			     = roota
                return roota
         return None







 @staticmethod
 def _snake_case (							a_							)		->   List[Any]:
        _UpperCAmelCase  :						int			     = graph.num_vertices

        _UpperCAmelCase  :						int			     = Graph.UnionFind()
        _UpperCAmelCase  :						Optional[int]			     = []
        while num_components > 1:
               _UpperCAmelCase  :						int			     = {}
               for vertex in graph.get_vertices():
                      _UpperCAmelCase  :						Union[str, Any]			     = -1

               _UpperCAmelCase  :						Tuple			     = graph.get_edges()
               for edge in edges:
                      _UpperCAmelCase     ,_UpperCAmelCase     ,_UpperCAmelCase  :						str			     = edge
                      edges.remove((tail, head, weight)							)
               for edge in edges:
                      _UpperCAmelCase     ,_UpperCAmelCase     ,_UpperCAmelCase  :						Optional[Any]			     = edge
                      _UpperCAmelCase  :						Any			     = union_find.find(a_							)
                      _UpperCAmelCase  :						Any			     = union_find.find(a_							)
                      if seta != seta:
                             if cheap_edge[seta] == -1 or cheap_edge[seta][2] > weight:
                                    _UpperCAmelCase  :						Tuple			     = [head, tail, weight]

                             if cheap_edge[seta] == -1 or cheap_edge[seta][2] > weight:
                                    _UpperCAmelCase  :						List[str]			     = [head, tail, weight]
               for vertex in cheap_edge:
                      if cheap_edge[vertex] != -1:
                             _UpperCAmelCase     ,_UpperCAmelCase     ,_UpperCAmelCase  :						str			     = cheap_edge[vertex]
                             if union_find.find(a_							) != union_find.find(a_							):
                                    union_find.union(a_  ,a_							)
                                    mst_edges.append(cheap_edge[vertex]							)
                                    _UpperCAmelCase  :						Tuple			     = num_components - 1
        _UpperCAmelCase  :						Optional[int]			     = Graph.build(edges=a_							)
        return mst



