


'''simple docstring'''
import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__    					=      get_tests_dir('''fixtures/test_sentencepiece.model''')




@require_sentencepiece
@require_tokenizers
class 			lowerCamelCase_   (	__a     ,       unittest.TestCase      ):
      lowerCAmelCase__		     =     XLNetTokenizer
      lowerCAmelCase__		     =     XLNetTokenizerFast
      lowerCAmelCase__		     =     True
      lowerCAmelCase__		     =     True






      def lowercase_							(					self				:    List[Any] ):


          '''simple docstring'''


          super().setUp()

          # We have a SentencePiece fixture for testing
          UpperCAmelCase__			:		Tuple													=				XLNetTokenizer(_A						,						keep_accents=_A )
          tokenizer.sanitize_special_tokens()
          tokenizer.save_pretrained(self.tmpdirname )






      def lowercase_							(					self				:    Tuple ):


          '''simple docstring'''


          UpperCAmelCase__			:		Optional[int]													=				'''<s>'''
          UpperCAmelCase__			:		List[str]													=				1

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A )						,						_A )
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A )						,						_A )






      def lowercase_							(					self				:    str ):


          '''simple docstring'''


          UpperCAmelCase__			:		Optional[Any]													=				list(self.get_tokenizer().get_vocab().keys() )

          self.assertEqual(vocab_keys[0]						,						'''<unk>''' )
          self.assertEqual(vocab_keys[1]						,						'''<s>''' )
          self.assertEqual(vocab_keys[-1]						,						'''<eod>''' )
          self.assertEqual(len(_A )						,						1_006 )






      def lowercase_							(					self				:    Any ):


          '''simple docstring'''


          self.assertEqual(self.get_tokenizer().vocab_size						,						1_000 )






      def lowercase_							(					self				:    Optional[int] ):


          '''simple docstring'''


          UpperCAmelCase__			:		int													=				XLNetTokenizer(_A						,						keep_accents=_A )

          UpperCAmelCase__			:		int													=				tokenizer.tokenize('''This is a test''' )
          self.assertListEqual(_A						,						['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''] )

          self.assertListEqual(tokenizer.convert_tokens_to_ids(_A )						,						[285, 46, 10, 170, 382] )

          UpperCAmelCase__			:		str													=				tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
          self.assertListEqual(
              _A						,						[
                  SPIECE_UNDERLINE + '''I''',
                  SPIECE_UNDERLINE + '''was''',
                  SPIECE_UNDERLINE + '''b''',
                  '''or''',
                  '''n''',
                  SPIECE_UNDERLINE + '''in''',
                  SPIECE_UNDERLINE + '''''',
                  '''9''',
                  '''2''',
                  '''0''',
                  '''0''',
                  '''0''',
                  ''',''',
                  SPIECE_UNDERLINE + '''and''',
                  SPIECE_UNDERLINE + '''this''',
                  SPIECE_UNDERLINE + '''is''',
                  SPIECE_UNDERLINE + '''f''',
                  '''al''',
                  '''s''',
                  '''é''',
                  '''.''',
              ]						,						)
          UpperCAmelCase__			:		Union[str, Any]													=				tokenizer.convert_tokens_to_ids(_A )
          self.assertListEqual(_A						,						[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4] )

          UpperCAmelCase__			:		List[str]													=				tokenizer.convert_ids_to_tokens(_A )
          self.assertListEqual(
              _A						,						[
                  SPIECE_UNDERLINE + '''I''',
                  SPIECE_UNDERLINE + '''was''',
                  SPIECE_UNDERLINE + '''b''',
                  '''or''',
                  '''n''',
                  SPIECE_UNDERLINE + '''in''',
                  SPIECE_UNDERLINE + '''''',
                  '''<unk>''',
                  '''2''',
                  '''0''',
                  '''0''',
                  '''0''',
                  ''',''',
                  SPIECE_UNDERLINE + '''and''',
                  SPIECE_UNDERLINE + '''this''',
                  SPIECE_UNDERLINE + '''is''',
                  SPIECE_UNDERLINE + '''f''',
                  '''al''',
                  '''s''',
                  '''<unk>''',
                  '''.''',
              ]						,						)






      def lowercase_							(					self				:    Any ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[Any]													=				XLNetTokenizer(_A						,						do_lower_case=_A )
          UpperCAmelCase__			:		int													=				tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
          self.assertListEqual(
              _A						,						[
                  SPIECE_UNDERLINE + '''''',
                  '''i''',
                  SPIECE_UNDERLINE + '''was''',
                  SPIECE_UNDERLINE + '''b''',
                  '''or''',
                  '''n''',
                  SPIECE_UNDERLINE + '''in''',
                  SPIECE_UNDERLINE + '''''',
                  '''9''',
                  '''2''',
                  '''0''',
                  '''0''',
                  '''0''',
                  ''',''',
                  SPIECE_UNDERLINE + '''and''',
                  SPIECE_UNDERLINE + '''this''',
                  SPIECE_UNDERLINE + '''is''',
                  SPIECE_UNDERLINE + '''f''',
                  '''al''',
                  '''se''',
                  '''.''',
              ]						,						)
          self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )						,						['''▁he''', '''ll''', '''o'''] )






      def lowercase_							(					self				:    Any ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[str]													=				XLNetTokenizer(_A						,						do_lower_case=_A )
          UpperCAmelCase__			:		Union[str, Any]													=				tokenizer.tokenize('''I was born in 92000, and this is falsé.''' )
          self.assertListEqual(
              _A						,						[
                  SPIECE_UNDERLINE + '''I''',
                  SPIECE_UNDERLINE + '''was''',
                  SPIECE_UNDERLINE + '''b''',
                  '''or''',
                  '''n''',
                  SPIECE_UNDERLINE + '''in''',
                  SPIECE_UNDERLINE + '''''',
                  '''9''',
                  '''2''',
                  '''0''',
                  '''0''',
                  '''0''',
                  ''',''',
                  SPIECE_UNDERLINE + '''and''',
                  SPIECE_UNDERLINE + '''this''',
                  SPIECE_UNDERLINE + '''is''',
                  SPIECE_UNDERLINE + '''f''',
                  '''al''',
                  '''se''',
                  '''.''',
              ]						,						)






      @slow
      def lowercase_							(					self				:    Tuple ):


          '''simple docstring'''


          UpperCAmelCase__			:		int													=				XLNetTokenizer.from_pretrained('''xlnet-base-cased''' )

          UpperCAmelCase__			:		Optional[int]													=				tokenizer.encode('''sequence builders'''						,						add_special_tokens=_A )
          UpperCAmelCase__			:		int													=				tokenizer.encode('''multi-sequence build'''						,						add_special_tokens=_A )

          UpperCAmelCase__			:		Union[str, Any]													=				tokenizer.build_inputs_with_special_tokens(_A )
          UpperCAmelCase__			:		List[str]													=				tokenizer.build_inputs_with_special_tokens(_A						,						_A )

          assert encoded_sentence == text + [4, 3]
          assert encoded_pair == text + [4] + text_a + [4, 3]






      @slow
      def lowercase_							(					self				:    List[Any] ):


          '''simple docstring'''


          UpperCAmelCase__			:		str													=				{'''input_ids''': [[17, 21_442, 270, 17, 10, 14_645, 318, 34, 17, 4_546, 3_145, 787, 13, 7_752, 22_018, 23, 21, 17, 4_546, 3_145, 787, 13, 3_352, 14_431, 13, 5_500, 11, 1_176, 580, 13, 16_819, 4_797, 23, 17, 10, 17_135, 658, 19, 457, 7_932, 13, 184, 19, 3_154, 17_135, 6_468, 19, 1_404, 12_269, 19, 4_229, 5_356, 16_264, 46, 19, 17, 20_545, 10_395, 9, 9, 9, 11, 28, 6_421, 9_531, 20_729, 17, 10, 353, 17_022, 11, 21, 6_421, 9_531, 16_949, 17, 10, 11_509, 753, 11, 33, 95, 2_421, 7_385, 956, 14_431, 2_626, 25, 842, 7_385, 4_836, 21, 1_429, 2_272, 9_855, 3_120, 161, 24_738, 19, 13_203, 658, 218, 787, 21, 430, 18_482, 847, 2_637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22_178, 27, 1_064, 22, 956, 13, 11_101, 1_429, 5_854, 24_313, 18_953, 40, 422, 24_366, 68, 1_758, 37, 10_483, 14_257, 31, 207, 263, 21, 203, 3_773, 25, 71, 9_735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2_049, 3_442, 17, 13_894, 3_380, 23, 95, 18, 17_634, 2_288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=_A						,						model_name='''xlnet-base-cased'''						,						revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''						,						)




'''simple docstring'''
# Algorithm for the pigeonhole sorting







def 					a__       (					lowerCAmelCase__						)    ->	Optional[Any]:
    UpperCAmelCase__			:		Any													=				min(lowerCAmelCase__						)  # min() finds the minimum value
    UpperCAmelCase__			:		Optional[int]													=				max(lowerCAmelCase__						)  # max() finds the maximum value

    UpperCAmelCase__			:		int													=				max_val - min_val + 1  # size is difference of max and min values plus one

    # list of pigeonholes of size equal to the variable size
    UpperCAmelCase__			:		Any													=				[0] * size

    # Populate the pigeonholes.
    for x in a:
        assert isinstance(lowerCAmelCase__  ,					lowerCAmelCase__						), "integers only please"
        holes[x - min_val] += 1

    # Putting the elements back into the array in an order.
    UpperCAmelCase__			:		Optional[int]													=				0
    for count in range(lowerCAmelCase__						):
        while holes[count] > 0:
            holes[count] -= 1
            UpperCAmelCase__			:		Dict													=				count + min_val
            i += 1





def 					a__       (					)    ->	Union[str, Any]:
    UpperCAmelCase__			:		List[str]													=				[8, 3, 2, 7, 4, 6, 8]
    pigeonhole_sort(lowerCAmelCase__						)
    print('''Sorted order is:'''  ,					''' '''.join(lowerCAmelCase__						)						)


if __name__ == "__main__":
  main()
