
'''simple docstring'''







import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowercase    :							Any      =					get_tests_dir("fixtures/test_sentencepiece.model")


@require_sentencepiece
class 						__UpperCAmelCase      (					_lowerCamelCase						,   unittest.TestCase						):
							__lowercase      		=       XLMProphetNetTokenizer
							__lowercase      		=       False
							__lowercase      		=       True


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								super().setUp()

								# We have a SentencePiece fixture for testing
								_snake_case   						=		XLMProphetNetTokenizer(lowerCAmelCase_    ,   keep_accents=lowerCAmelCase_  )
								tokenizer.save_pretrained(self.tmpdirname  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		'[PAD]'
								_snake_case   						=		0

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase_  )    ,   lowerCAmelCase_  )
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase_  )    ,   lowerCAmelCase_  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		list(self.get_tokenizer().get_vocab().keys()  )

								self.assertEqual(vocab_keys[0]    ,   '[PAD]'  )
								self.assertEqual(vocab_keys[1]    ,   '[CLS]'  )
								self.assertEqual(vocab_keys[-1]    ,   'j'  )
								self.assertEqual(len(lowerCAmelCase_  )    ,   10_12  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								self.assertEqual(self.get_tokenizer().vocab_size    ,   10_12  )


							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		XLMProphetNetTokenizer(lowerCAmelCase_    ,   keep_accents=lowerCAmelCase_  )

								_snake_case   						=		tokenizer.tokenize('This is a test'  )
								self.assertListEqual(lowerCAmelCase_    ,   ['▁This', '▁is', '▁a', '▁t', 'est']  )

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(lowerCAmelCase_  )    ,   [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]    ,   )

								_snake_case   						=		tokenizer.tokenize('I was born in 92000, and this is falsé.'  )
								self.assertListEqual(
								    lowerCAmelCase_    ,   [
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '9',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        'é',
								        '.',
								    ]    ,   )
								_snake_case   						=		tokenizer.convert_tokens_to_ids(lowerCAmelCase_  )
								self.assertListEqual(
								    lowerCAmelCase_    ,   [
								        value + tokenizer.fairseq_offset
								        for value in [8, 21, 84, 55, 24, 19, 7, -9, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, -9, 4]
								    ]    ,   )

								_snake_case   						=		tokenizer.convert_ids_to_tokens(lowerCAmelCase_  )
								self.assertListEqual(
								    lowerCAmelCase_    ,   [
								        SPIECE_UNDERLINE + 'I',
								        SPIECE_UNDERLINE + 'was',
								        SPIECE_UNDERLINE + 'b',
								        'or',
								        'n',
								        SPIECE_UNDERLINE + 'in',
								        SPIECE_UNDERLINE + '',
								        '[UNK]',
								        '2',
								        '0',
								        '0',
								        '0',
								        ',',
								        SPIECE_UNDERLINE + 'and',
								        SPIECE_UNDERLINE + 'this',
								        SPIECE_UNDERLINE + 'is',
								        SPIECE_UNDERLINE + 'f',
								        'al',
								        's',
								        '[UNK]',
								        '.',
								    ]    ,   )


							@cached_property
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								return XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased'  )


							@slow
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		'Hello World!'
								_snake_case   						=		[3_53_89, 66_72, 49, 2]
								self.assertListEqual(lowerCAmelCase_    ,   self.big_tokenizer.encode(lowerCAmelCase_  )  )




							@slow
							def  lowerCamelCase				(       self  ):





								"""simple docstring"""






								_snake_case   						=		{'input_ids': [[1_10_73, 8_27_83, 18, 26, 8_27_83, 5_49, 5_15_40, 2_48, 1_72_09, 13_01, 2_17, 20, 21_51_86, 13_25, 1_47, 1_72_09, 13_01, 2_17, 20, 5_63_70, 53, 12_20_20, 20, 1_64_77, 27, 8_73_55, 45_48, 20, 47_28, 7_83_92, 17, 15_99_69, 18, 26, 2_44_91, 6_29, 15, 5_38, 2_27_04, 54_39, 15, 27_88, 2_44_91, 98_85, 15, 4_35_34, 6_05, 15, 8_14, 1_84_03, 3_32_00, 29, 15, 4_35_34, 2_44_58, 1_24_10, 1_11, 2_49_66, 8_36_69, 96_37, 14_40_68, 26, 8_50, 2_23_46, 27, 1_47, 2_49_66, 8_36_69, 8_34_90, 26, 3_91_13, 7_35, 27, 6_89, 6_56, 28_00, 13_39, 46_00, 53, 12_20_20, 11_57_85, 34, 8_16, 13_39, 4_68_87, 18, 1_47, 5_39_05, 19_51, 4_22_38, 4_11_70, 1_77_32, 8_34, 4_36, 15, 2_75_23, 9_87_33, 2_17, 1_47, 55_42, 49_81, 9_30, 1_73_47, 16, 2], [2_00_91, 6_29, 94, 8_27_86, 58, 4_90, 20, 15_28, 84, 5_39_05, 3_44, 8_05_92, 11_01_28, 1_88_22, 52_67, 13_06, 62, 15_25_37, 3_08, 79_97, 4_01, 12_44_27, 5_49, 3_54_42, 2_25, 1_09, 1_50_55, 2_57_48, 1_47, 71_19, 4_37_12, 34, 7_67, 13_53_66, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_92, 6_37_84, 11_94_66, 17, 14_78_08, 8_82_14, 18, 6_56, 81, 32, 32_96, 1_02_80, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=lowerCAmelCase_    ,   model_name='microsoft/xprophetnet-large-wiki100-cased'    ,   revision='1acad1643ddd54a44df6a1b797ada8373685d90e'    ,   )





"""simple docstring"""



def   __lowerCAmelCase						(						lowercase  :  List[str]						,							lowercase  :  Union[str, Any]						,							lowercase  :  List[str]						,							lowercase  :  Tuple						,							lowercase  :  List[Any]						,							lowercase  :  int )	->						List[Any]:


    """simple docstring"""





    if index == r:
        for j in range(lowercase ):
            print(data[j]						,							end=" " )
        print(" " )
        return
    #  When no more elements are there to put in data[]
    if i >= n:
        return
    # current is included, put next at next location
    snake_case    :      Union[str, Any]      					=    arr[i]
    combination_util(lowercase						,							lowercase						,							lowercase						,							index + 1						,							lowercase						,							i + 1 )
    # current is excluded, replace it with
    # next (Note that i+1 is passed, but
    # index is not changed)
    combination_util(lowercase						,							lowercase						,							lowercase						,							lowercase						,							lowercase						,							i + 1 )
    # The main function that prints all combinations
    # of size r in arr[] of size n. This function
    # mainly uses combinationUtil()







def   __lowerCAmelCase						(						lowercase  :  Any						,							lowercase  :  Union[str, Any]						,							lowercase  :  Optional[int] )	->						List[Any]:


    """simple docstring"""





    snake_case    :      Any      					=    [0] * r
    # Print all combination using temporary array 'data[]'
    combination_util(lowercase						,							lowercase						,							lowercase						,							0						,							lowercase						,							0 )


if __name__ == "__main__":
  # Driver code to check the function above
  __snake_case   			=    [10, 20, 30, 40, 50]
  print_combination(arr, len(arr), 3)
  # This code is contributed by Ambuj sahu





