"""simple docstring"""





import itertools
import random
import unittest

import numpy as np

from transformers import is_speech_available
from transformers.testing_utils import require_torch, require_torchaudio

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
				from transformers import SpeechaTextFeatureExtractor

__A     = random.Random()


def       lowercase_      (      _lowerCamelCase:		Dict  ,      _lowerCamelCase:		List[str]=1.0  ,      _lowerCamelCase:		str=None  ,      _lowerCamelCase:		List[str]=None					)       ->					Tuple:



							'''simple docstring'''

							if rng is None:
														__lowerCamelCase    :						Optional[Any]						      =   global_rng

							__lowerCamelCase    :						Tuple						      =   []
							for batch_idx in range(shape[0]					):
														values.append([]					)
														for _ in range(shape[1]					):
																					values[-1].append(rng.random() * scale					)

							return values
@require_torch
@require_torchaudio
class        _snake_case   (    unittest.TestCase						):






						def __init__(				self					:	Optional[Any]							,    UpperCAmelCase					:	Dict							,    UpperCAmelCase					:	Optional[int]=7							,    UpperCAmelCase					:	Any=400							,    UpperCAmelCase					:	List[str]=2000							,    UpperCAmelCase					:	int=24							,    UpperCAmelCase					:	Union[str, Any]=24							,    UpperCAmelCase					:	Dict=0.0							,    UpperCAmelCase					:	Optional[Any]=16000							,    UpperCAmelCase					:	int=True							,    UpperCAmelCase					:	Union[str, Any]=True							,    ):
													__lowerCamelCase    :						List[Any]						      =   parent
													__lowerCamelCase    :						Dict						      =   batch_size
													__lowerCamelCase    :						int						      =   min_seq_length
													__lowerCamelCase    :						Tuple						      =   max_seq_length
													__lowerCamelCase    :						Optional[int]						      =   (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
													__lowerCamelCase    :						Optional[int]						      =   feature_size
													__lowerCamelCase    :						str						      =   num_mel_bins
													__lowerCamelCase    :						Tuple						      =   padding_value
													__lowerCamelCase    :						Any						      =   sampling_rate
													__lowerCamelCase    :						int						      =   return_attention_mask
													__lowerCamelCase    :						List[str]						      =   do_normalize






						def 					lowerCamelCase__   (				self					:	Any			):
													return {
													    "feature_size": self.feature_size,
													    "num_mel_bins": self.num_mel_bins,
													    "padding_value": self.padding_value,
													    "sampling_rate": self.sampling_rate,
													    "return_attention_mask": self.return_attention_mask,
													    "do_normalize": self.do_normalize,
													}







						def 					lowerCamelCase__   (				self					:	Optional[int]							,    UpperCAmelCase					:	int=False							,    UpperCAmelCase					:	int=False			):




													def _flatten(UpperCAmelCase					:	str			):
																				return list(itertools.chain(*lowercase_			)			)

													if equal_length:
																				__lowerCamelCase    :						Optional[int]						      =   [floats_list((self.max_seq_length, self.feature_size)			) for _ in range(self.batch_size			)]
													else:
																				# make sure that inputs increase in size
																				__lowerCamelCase    :						Optional[Any]						      =   [
																				    floats_list((x, self.feature_size)			)
																				    for x in range(self.min_seq_length							,    self.max_seq_length							,    self.seq_length_diff			)
																				]
													if numpify:
																				__lowerCamelCase    :						Optional[int]						      =   [np.asarray(lowercase_			) for x in speech_inputs]
													return speech_inputs

@require_torch
@require_torchaudio
class        _snake_case   (    UpperCamelCase_ ,       unittest.TestCase						):
						snake_case__	     =	SpeechaTextFeatureExtractor if is_speech_available() else None






						def 					lowerCamelCase__   (				self					:	Union[str, Any]			):
													__lowerCamelCase    :						Optional[Any]						      =   SpeechaTextFeatureExtractionTester(self			)






						def 					lowerCamelCase__   (				self					:	Tuple							,    UpperCAmelCase					:	int			):
													self.assertTrue(np.all(np.mean(lowercase_							,    axis=0			) < 1E-3			)			)
													self.assertTrue(np.all(np.abs(np.var(lowercase_							,    axis=0			) - 1			) < 1E-3			)			)






						def 					lowerCamelCase__   (				self					:	Optional[int]			):
													__lowerCamelCase    :						Optional[Any]						      =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
													# create three inputs of length 800, 1000, and 1200
													__lowerCamelCase    :						Dict						      =   [floats_list((1, x)			)[0] for x in range(800							,    1400							,    200			)]
													__lowerCamelCase    :						Optional[int]						      =   [np.asarray(lowercase_			) for speech_input in speech_inputs]

													# Test feature size
													__lowerCamelCase    :						str						      =   feature_extractor(lowercase_							,    padding=lowercase_							,    return_tensors="np"			).input_features
													self.assertTrue(input_features.ndim == 3			)
													self.assertTrue(input_features.shape[-1] == feature_extractor.feature_size			)

													# Test not batched input
													__lowerCamelCase    :						Optional[Any]						      =   feature_extractor(speech_inputs[0]							,    return_tensors="np"			).input_features
													__lowerCamelCase    :						List[str]						      =   feature_extractor(np_speech_inputs[0]							,    return_tensors="np"			).input_features
													self.assertTrue(np.allclose(lowercase_							,    lowercase_							,    atol=1E-3			)			)

													# Test batched
													__lowerCamelCase    :						int						      =   feature_extractor(lowercase_							,    return_tensors="np"			).input_features
													__lowerCamelCase    :						Optional[Any]						      =   feature_extractor(lowercase_							,    return_tensors="np"			).input_features
													for enc_seq_a, enc_seq_a in zip(lowercase_							,    lowercase_			):
																				self.assertTrue(np.allclose(lowercase_							,    lowercase_							,    atol=1E-3			)			)

													# Test 2-D numpy arrays are batched.
													__lowerCamelCase    :						Optional[int]						      =   [floats_list((1, x)			)[0] for x in (800, 800, 800)]
													__lowerCamelCase    :						Union[str, Any]						      =   np.asarray(lowercase_			)
													__lowerCamelCase    :						Any						      =   feature_extractor(lowercase_							,    return_tensors="np"			).input_features
													__lowerCamelCase    :						Tuple						      =   feature_extractor(lowercase_							,    return_tensors="np"			).input_features
													for enc_seq_a, enc_seq_a in zip(lowercase_							,    lowercase_			):
																				self.assertTrue(np.allclose(lowercase_							,    lowercase_							,    atol=1E-3			)			)






						def 					lowerCamelCase__   (				self					:	Tuple			):
													__lowerCamelCase    :						Any						      =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
													__lowerCamelCase    :						List[Any]						      =   [floats_list((1, x)			)[0] for x in range(800							,    1400							,    200			)]

													__lowerCamelCase    :						int						      =   ['longest', 'max_length', 'do_not_pad']
													__lowerCamelCase    :						Any						      =   [None, 16, None]
													for max_length, padding in zip(lowercase_							,    lowercase_			):
																				__lowerCamelCase    :						List[str]						      =   feature_extractor(
																				    lowercase_							,    padding=lowercase_							,    max_length=lowercase_							,    return_attention_mask=lowercase_			)
																				__lowerCamelCase    :						Union[str, Any]						      =   inputs.input_features
																				__lowerCamelCase    :						List[Any]						      =   inputs.attention_mask
																				__lowerCamelCase    :						Union[str, Any]						      =   [np.sum(lowercase_			) for x in attention_mask]

																				self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]]			)
																				self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]]			)
																				self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]]			)






						def 					lowerCamelCase__   (				self					:	Optional[Any]			):
													__lowerCamelCase    :						Union[str, Any]						      =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
													__lowerCamelCase    :						Any						      =   [floats_list((1, x)			)[0] for x in range(800							,    1400							,    200			)]

													__lowerCamelCase    :						Optional[int]						      =   ['longest', 'max_length', 'do_not_pad']
													__lowerCamelCase    :						Any						      =   [None, 16, None]
													for max_length, padding in zip(lowercase_							,    lowercase_			):
																				__lowerCamelCase    :						Optional[Any]						      =   feature_extractor(
																				    lowercase_							,    max_length=lowercase_							,    padding=lowercase_							,    return_tensors="np"							,    return_attention_mask=lowercase_			)
																				__lowerCamelCase    :						Union[str, Any]						      =   inputs.input_features
																				__lowerCamelCase    :						Optional[int]						      =   inputs.attention_mask
																				__lowerCamelCase    :						List[Any]						      =   [np.sum(lowercase_			) for x in attention_mask]

																				self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]]			)
																				self.assertTrue(input_features[0][fbank_feat_lengths[0] :].sum() < 1E-6			)
																				self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]]			)
																				self.assertTrue(input_features[0][fbank_feat_lengths[1] :].sum() < 1E-6			)
																				self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]]			)






						def 					lowerCamelCase__   (				self					:	Union[str, Any]			):
													__lowerCamelCase    :						Optional[int]						      =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
													__lowerCamelCase    :						Tuple						      =   [floats_list((1, x)			)[0] for x in range(800							,    1400							,    200			)]
													__lowerCamelCase    :						Union[str, Any]						      =   feature_extractor(
													    lowercase_							,    padding="max_length"							,    max_length=4							,    truncation=lowercase_							,    return_tensors="np"							,    return_attention_mask=lowercase_							,    )
													__lowerCamelCase    :						Union[str, Any]						      =   inputs.input_features
													__lowerCamelCase    :						Dict						      =   inputs.attention_mask
													__lowerCamelCase    :						str						      =   np.sum(attention_mask == 1							,    axis=1			)

													self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]			)
													self._check_zero_mean_unit_variance(input_features[1]			)
													self._check_zero_mean_unit_variance(input_features[2]			)






						def 					lowerCamelCase__   (				self					:	Union[str, Any]			):
													__lowerCamelCase    :						List[Any]						      =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
													__lowerCamelCase    :						List[Any]						      =   [floats_list((1, x)			)[0] for x in range(800							,    1400							,    200			)]
													__lowerCamelCase    :						Optional[int]						      =   feature_extractor(
													    lowercase_							,    padding="longest"							,    max_length=4							,    truncation=lowercase_							,    return_tensors="np"							,    return_attention_mask=lowercase_							,    )
													__lowerCamelCase    :						List[str]						      =   inputs.input_features
													__lowerCamelCase    :						Optional[int]						      =   inputs.attention_mask
													__lowerCamelCase    :						Optional[Any]						      =   np.sum(attention_mask == 1							,    axis=1			)

													self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]			)
													self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]]			)
													self._check_zero_mean_unit_variance(input_features[2]			)

													# make sure that if max_length < longest -> then pad to max_length
													self.assertEqual(input_features.shape							,    (3, 4, 24)			)

													__lowerCamelCase    :						str						      =   [floats_list((1, x)			)[0] for x in range(800							,    1400							,    200			)]
													__lowerCamelCase    :						int						      =   feature_extractor(
													    lowercase_							,    padding="longest"							,    max_length=16							,    truncation=lowercase_							,    return_tensors="np"							,    return_attention_mask=lowercase_							,    )
													__lowerCamelCase    :						int						      =   inputs.input_features
													__lowerCamelCase    :						List[str]						      =   inputs.attention_mask
													__lowerCamelCase    :						int						      =   np.sum(attention_mask == 1							,    axis=1			)

													self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]			)
													self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]]			)
													self._check_zero_mean_unit_variance(input_features[2]			)

													# make sure that if max_length < longest -> then pad to max_length
													self.assertEqual(input_features.shape							,    (3, 6, 24)			)






						def 					lowerCamelCase__   (				self					:	List[str]			):
													import torch

													__lowerCamelCase    :						Union[str, Any]						      =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
													__lowerCamelCase    :						Optional[int]						      =   np.random.rand(100							,    32			).astype(np.floataa			)
													__lowerCamelCase    :						Tuple						      =   np_speech_inputs.tolist()

													for inputs in [py_speech_inputs, np_speech_inputs]:
																				__lowerCamelCase    :						List[Any]						      =   feature_extractor.pad([{"input_features": inputs}]							,    return_tensors="np"			)
																				self.assertTrue(np_processed.input_features.dtype == np.floataa			)
																				__lowerCamelCase    :						Any						      =   feature_extractor.pad([{"input_features": inputs}]							,    return_tensors="pt"			)
																				self.assertTrue(pt_processed.input_features.dtype == torch.floataa			)






						def 					lowerCamelCase__   (				self					:	Any							,    UpperCAmelCase					:	Tuple			):
													from datasets import load_dataset

													__lowerCamelCase    :						Union[str, Any]						      =   load_dataset("hf-internal-testing/librispeech_asr_dummy"							,    "clean"							,    split="validation"			)
													# automatic decoding with librispeech
													__lowerCamelCase    :						Tuple						      =   ds.sort("id"			).select(range(lowercase_			)			)[:num_samples]['audio']

													return [x["array"] for x in speech_samples]







						def 					lowerCamelCase__   (				self					:	Dict			):
													__lowerCamelCase    :						Optional[Any]						      =   np.array([
													    -1.5_7_4_5, -1.7_7_1_3, -1.7_0_2_0, -1.6_0_6_9, -1.2_2_5_0, -1.1_1_0_5, -0.9_0_7_2, -0.8_2_4_1,
													    -1.2_3_1_0, -0.8_0_9_8, -0.3_3_2_0, -0.4_1_0_1, -0.7_9_8_5, -0.4_9_9_6, -0.8_2_1_3, -0.9_1_2_8,
													    -1.0_4_2_0, -1.1_2_8_6, -1.0_4_4_0, -0.7_9_9_9, -0.8_4_0_5, -1.2_2_7_5, -1.5_4_4_3, -1.4_6_2_5,
													]			)
													# fmt: on

													__lowerCamelCase    :						str						      =   self._load_datasamples(1			)
													__lowerCamelCase    :						Optional[int]						      =   self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()			)
													__lowerCamelCase    :						Union[str, Any]						      =   feature_extractor(lowercase_							,    return_tensors="pt"			).input_features
													self.assertEquals(input_features.shape							,    (1, 584, 24)			)
													self.assertTrue(np.allclose(input_features[0, 0, :30]							,    lowercase_							,    atol=1E-4			)			)



import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase											=       get_tests_dir('fixtures/test_sentencepiece.model')




@require_sentencepiece
class A  (   UpperCamelCase_    ,							unittest.TestCase     ):
      UpperCamelCase__							:  str      =XLMProphetNetTokenizer
      UpperCamelCase__							:  Any      =False
      UpperCamelCase__							:  Optional[Any]      =True






      def        lowerCamelCase       (					self						:     Tuple )					->						Optional[Any]:
             """simple docstring"""




             super().setUp()

             # We have a SentencePiece fixture for testing
             _lowerCamelCase       :		Union[str, Any]		=XLMProphetNetTokenizer(lowercase_			,   keep_accents=lowercase_ )
             tokenizer.save_pretrained(self.tmpdirname )






      def        lowerCamelCase       (					self						:     int )					->						List[str]:
             """simple docstring"""




             _lowerCamelCase       :		Tuple		='[PAD]'
             _lowerCamelCase       :		Dict		=0

             self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_ )			,   lowercase_ )
             self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_ )			,   lowercase_ )






      def        lowerCamelCase       (					self						:     Optional[Any] )					->						int:
             """simple docstring"""




             _lowerCamelCase       :		int		=list(self.get_tokenizer().get_vocab().keys() )

             self.assertEqual(vocab_keys[0]			,   '[PAD]' )
             self.assertEqual(vocab_keys[1]			,   '[CLS]' )
             self.assertEqual(vocab_keys[-1]			,   'j' )
             self.assertEqual(len(lowercase_ )			,   1012 )






      def        lowerCamelCase       (					self						:     Optional[int] )					->						Any:
             """simple docstring"""




             self.assertEqual(self.get_tokenizer().vocab_size			,   1012 )






      def        lowerCamelCase       (					self						:     Dict )					->						Union[str, Any]:
             """simple docstring"""




             _lowerCamelCase       :		List[Any]		=XLMProphetNetTokenizer(lowercase_			,   keep_accents=lowercase_ )

             _lowerCamelCase       :		Any		=tokenizer.tokenize('This is a test' )
             self.assertListEqual(lowercase_			,   ['▁This', '▁is', '▁a', '▁t', 'est'] )

             self.assertListEqual(
                 tokenizer.convert_tokens_to_ids(lowercase_ )			,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,   )

             _lowerCamelCase       :		Dict		=tokenizer.tokenize('I was born in 92000, and this is falsé.' )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '9',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     'é',
                     '.',
                 ]			,   )
             _lowerCamelCase       :		Union[str, Any]		=tokenizer.convert_tokens_to_ids(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     value + tokenizer.fairseq_offset
                     for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
                 ]			,   )

             _lowerCamelCase       :		int		=tokenizer.convert_ids_to_tokens(lowercase_ )
             self.assertListEqual(
                 lowercase_			,   [
                     SPIECE_UNDERLINE + 'I',
                     SPIECE_UNDERLINE + 'was',
                     SPIECE_UNDERLINE + 'b',
                     'or',
                     'n',
                     SPIECE_UNDERLINE + 'in',
                     SPIECE_UNDERLINE + '',
                     '[UNK]',
                     '2',
                     '0',
                     '0',
                     '0',
                     ',',
                     SPIECE_UNDERLINE + 'and',
                     SPIECE_UNDERLINE + 'this',
                     SPIECE_UNDERLINE + 'is',
                     SPIECE_UNDERLINE + 'f',
                     'al',
                     's',
                     '[UNK]',
                     '.',
                 ]			,   )






      @cached_property
      def        lowerCamelCase       (					self						:     Any )					->						Optional[Any]:
             """simple docstring"""




             return XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased' )






      @slow
      def        lowerCamelCase       (					self						:     List[str] )					->						Any:
             """simple docstring"""




             _lowerCamelCase       :		Optional[int]		='Hello World!'
             _lowerCamelCase       :		Optional[int]		=[3_5389, 6672, 49, 2]
             self.assertListEqual(lowercase_			,   self.big_tokenizer.encode(lowercase_ ) )






      @slow
      def        lowerCamelCase       (					self						:     Optional[int] )					->						Dict:
             """simple docstring"""




             _lowerCamelCase       :		Dict		={'input_ids': [[1_1073, 8_2783, 18, 26, 8_2783, 549, 5_1540, 248, 1_7209, 1301, 217, 20, 21_5186, 1325, 147, 1_7209, 1301, 217, 20, 5_6370, 53, 12_2020, 20, 1_6477, 27, 8_7355, 4548, 20, 4728, 7_8392, 17, 15_9969, 18, 26, 2_4491, 629, 15, 538, 2_2704, 5439, 15, 2788, 2_4491, 9885, 15, 4_3534, 605, 15, 814, 1_8403, 3_3200, 29, 15, 4_3534, 2_4458, 1_2410, 111, 2_4966, 8_3669, 9637, 14_4068, 26, 850, 2_2346, 27, 147, 2_4966, 8_3669, 8_3490, 26, 3_9113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 12_2020, 11_5785, 34, 816, 1339, 4_6887, 18, 147, 5_3905, 1951, 4_2238, 4_1170, 1_7732, 834, 436, 15, 2_7523, 9_8733, 217, 147, 5542, 4981, 930, 1_7347, 16, 2], [2_0091, 629, 94, 8_2786, 58, 490, 20, 1528, 84, 5_3905, 344, 8_0592, 11_0128, 1_8822, 5267, 1306, 62, 15_2537, 308, 7997, 401, 12_4427, 549, 3_5442, 225, 109, 1_5055, 2_5748, 147, 7119, 4_3712, 34, 767, 13_5366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 6_3784, 11_9466, 17, 14_7808, 8_8214, 18, 656, 81, 32, 3296, 1_0280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
             # fmt: on

             self.tokenizer_integration_test_util(
                 expected_encoding=lowercase_			,   model_name='microsoft/xprophetnet-large-wiki100-cased'			,   revision='1acad1643ddd54a44df6a1b797ada8373685d90e'			,   )


