



import numpy as np
import torch
from torch.utils.data import Dataset

from utils import logger


class       a							(			__lowerCamelCase				):



		def __init__(     self     :str   ,__lowercase     :int   ,__lowercase     :Optional[Any]  ):
						snake_case__	:      List[Any]     		=	params

						snake_case__	:      int     		=	np.array(__lowercase  )
						snake_case__	:      str     		=	np.array([len(__lowercase  ) for t in data]  )

						self.check()
						self.remove_long_sequences()
						self.remove_empty_sequences()
						self.remove_unknown_sequences()
						self.check()
						self.print_statistics()



		def __getitem__(     self     :str   ,__lowercase     :List[Any]  ):
						return (self.token_ids[index], self.lengths[index])



		def __len__(     self     :Dict  ):
						return len(self.lengths  )



		def 	__lowerCamelCase			(     self     :List[Any]  ):
						assert len(self.token_ids  ) == len(self.lengths  )
						assert all(self.lengths[i] == len(self.token_ids[i]  ) for i in range(len(self.lengths  )  )  )



		def 	__lowerCamelCase			(     self     :int  ):
						snake_case__	:      List[Any]     		=	self.params.max_model_input_size
						snake_case__	:      Any     		=	self.lengths > max_len
						logger.info(F"""Splitting {sum(__lowercase  )} too long sequences."""  )

						def divide_chunks(__lowercase     :Dict   ,__lowercase     :List[str]  ):
										return [l[i : i + n] for i in range(0   ,len(__lowercase  )   ,__lowercase  )]

						snake_case__	:      List[Any]     		=	[]
						snake_case__	:      Union[str, Any]     		=	[]
						if self.params.mlm:
										snake_case__		, snake_case__	:      str     		=	self.params.special_tok_ids['''cls_token'''], self.params.special_tok_ids['''sep_token''']
						else:
										snake_case__		, snake_case__	:      Optional[int]     		=	self.params.special_tok_ids['''bos_token'''], self.params.special_tok_ids['''eos_token''']

						for seq_, len_ in zip(self.token_ids   ,self.lengths  ):
										assert (seq_[0] == cls_id) and (seq_[-1] == sep_id), seq_
										if len_ <= max_len:
														new_tok_ids.append(seq_  )
														new_lengths.append(len_  )
										else:
														snake_case__	:      Optional[Any]     		=	[]
														for sub_s in divide_chunks(seq_   ,max_len - 2  ):
																		if sub_s[0] != cls_id:
																						snake_case__	:      Union[str, Any]     		=	np.insert(__lowercase   ,0   ,__lowercase  )
																		if sub_s[-1] != sep_id:
																						snake_case__	:      Optional[Any]     		=	np.insert(__lowercase   ,len(__lowercase  )   ,__lowercase  )
																		assert len(__lowercase  ) <= max_len
																		assert (sub_s[0] == cls_id) and (sub_s[-1] == sep_id), sub_s
																		sub_seqs.append(__lowercase  )

														new_tok_ids.extend(__lowercase  )
														new_lengths.extend([len(__lowercase  ) for l in sub_seqs]  )

						snake_case__	:      int     		=	np.array(__lowercase  )
						snake_case__	:      List[Any]     		=	np.array(__lowercase  )



		def 	__lowerCamelCase			(     self     :int  ):
						snake_case__	:      Union[str, Any]     		=	len(self  )
						snake_case__	:      Any     		=	self.lengths > 1_1
						snake_case__	:      Union[str, Any]     		=	self.token_ids[indices]
						snake_case__	:      Union[str, Any]     		=	self.lengths[indices]
						snake_case__	:      Tuple     		=	len(self  )
						logger.info(F"""Remove {init_size - new_size} too short (<=11 tokens) sequences."""  )



		def 	__lowerCamelCase			(     self     :Any  ):
						if "unk_token" not in self.params.special_tok_ids:
										return
						else:
										snake_case__	:      int     		=	self.params.special_tok_ids['''unk_token''']
						snake_case__	:      Tuple     		=	len(self  )
						snake_case__	:      int     		=	np.array([np.count_nonzero(a == unk_token_id  ) for a in self.token_ids]  )
						snake_case__	:      Tuple     		=	(unk_occs / self.lengths) < 0.5
						snake_case__	:      List[Any]     		=	self.token_ids[indices]
						snake_case__	:      Tuple     		=	self.lengths[indices]
						snake_case__	:      List[Any]     		=	len(self  )
						logger.info(F"""Remove {init_size - new_size} sequences with a high level of unknown tokens (50%)."""  )



		def 	__lowerCamelCase			(     self     :int  ):
						if not self.params.is_master:
										return
						logger.info(F"""{len(self  )} sequences"""  )
						# data_len = sum(self.lengths)
						# nb_unique_tokens = len(Counter(list(chain(*self.token_ids))))
						# logger.info(f'{data_len} tokens ({nb_unique_tokens} unique)')

						# unk_idx = self.params.special_tok_ids['unk_token']
						# nb_unknown = sum([(t==unk_idx).sum() for t in self.token_ids])
						# logger.info(f'{nb_unknown} unknown tokens (covering {100*nb_unknown/data_len:.2f}% of the data)')






		def 	__lowerCamelCase			(     self     :int   ,__lowercase     :Optional[Any]  ):
						snake_case__	:      Optional[Any]     		=	[t[0] for t in batch]
						snake_case__	:      List[Any]     		=	[t[1] for t in batch]
						assert len(__lowercase  ) == len(__lowercase  )

						# Max for paddings
						snake_case__	:      Optional[int]     		=	max(__lowercase  )

						# Pad token ids
						if self.params.mlm:
										snake_case__	:      Any     		=	self.params.special_tok_ids['''pad_token''']
						else:
										snake_case__	:      List[str]     		=	self.params.special_tok_ids['''unk_token''']
						snake_case__	:      List[Any]     		=	[list(t.astype(__lowercase  )  ) + [pad_idx] * (max_seq_len_ - len(__lowercase  )) for t in token_ids]
						assert len(tk_  ) == len(__lowercase  )
						assert all(len(__lowercase  ) == max_seq_len_ for t in tk_  )

						snake_case__	:      List[str]     		=	torch.tensor(tk_  )  # (bs, max_seq_len_)
						snake_case__	:      int     		=	torch.tensor(__lowercase  )  # (bs)
						return tk_t, lg_t






import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY


@is_pipeline_test
class       a							(			unittest.TestCase				):
		__lowerCAmelCase	:    Any											=					MODEL_FOR_MASKED_LM_MAPPING
		__lowerCAmelCase	:    Optional[Any]											=					TF_MODEL_FOR_MASKED_LM_MAPPING



		def 	__lowerCamelCase			(     self     :str  ):
						super().tearDown()
						# clean-up as much as possible GPU memory occupied by PyTorch
						gc.collect()
						if is_torch_available():
										import torch

										torch.cuda.empty_cache()



		@require_tf
		def 	__lowerCamelCase			(     self     :Any  ):
						snake_case__	:      Optional[Any]     		=	pipeline(task='''fill-mask'''   ,model='''sshleifer/tiny-distilroberta-base'''   ,top_k=2   ,framework='''tf'''  )
						snake_case__	:      int     		=	unmasker('''My name is <mask>'''  )
						self.assertEqual(
						    nested_simplify(__lowercase   ,decimals=6  )   ,[
						        {'''sequence''': '''My name is grouped''', '''score''': 2.1e-0_5, '''token''': 3_8_0_1_5, '''token_str''': ''' grouped'''},
						        {'''sequence''': '''My name is accuser''', '''score''': 2.1e-0_5, '''token''': 2_5_5_0_6, '''token_str''': ''' accuser'''},
						    ]   ,)

						snake_case__	:      int     		=	unmasker('''The largest city in France is <mask>'''  )
						self.assertEqual(
						    nested_simplify(__lowercase   ,decimals=6  )   ,[
						        {
						            '''sequence''': '''The largest city in France is grouped''',
						            '''score''': 2.1e-0_5,
						            '''token''': 3_8_0_1_5,
						            '''token_str''': ''' grouped''',
						        },
						        {
						            '''sequence''': '''The largest city in France is accuser''',
						            '''score''': 2.1e-0_5,
						            '''token''': 2_5_5_0_6,
						            '''token_str''': ''' accuser''',
						        },
						    ]   ,)

						snake_case__	:      Optional[int]     		=	unmasker('''My name is <mask>'''   ,targets=[''' Patrick''', ''' Clara''', ''' Teven''']   ,top_k=3  )
						self.assertEqual(
						    nested_simplify(__lowercase   ,decimals=6  )   ,[
						        {'''sequence''': '''My name is Clara''', '''score''': 2e-0_5, '''token''': 1_3_6_0_6, '''token_str''': ''' Clara'''},
						        {'''sequence''': '''My name is Patrick''', '''score''': 2e-0_5, '''token''': 3_4_9_9, '''token_str''': ''' Patrick'''},
						        {'''sequence''': '''My name is Te''', '''score''': 1.9e-0_5, '''token''': 2_9_4_1, '''token_str''': ''' Te'''},
						    ]   ,)



		@require_torch
		def 	__lowerCamelCase			(     self     :Optional[int]  ):
						snake_case__	:      str     		=	pipeline(task='''fill-mask'''   ,model='''sshleifer/tiny-distilroberta-base'''   ,top_k=2   ,framework='''pt'''  )

						snake_case__	:      str     		=	unmasker('''My name is <mask>'''  )
						self.assertEqual(
						    nested_simplify(__lowercase   ,decimals=6  )   ,[
						        {'''sequence''': '''My name is Maul''', '''score''': 2.2e-0_5, '''token''': 3_5_6_7_6, '''token_str''': ''' Maul'''},
						        {'''sequence''': '''My name isELS''', '''score''': 2.2e-0_5, '''token''': 1_6_4_1_6, '''token_str''': '''ELS'''},
						    ]   ,)

						snake_case__	:      List[str]     		=	unmasker('''The largest city in France is <mask>'''  )
						self.assertEqual(
						    nested_simplify(__lowercase   ,decimals=6  )   ,[
						        {
						            '''sequence''': '''The largest city in France is Maul''',
						            '''score''': 2.2e-0_5,
						            '''token''': 3_5_6_7_6,
						            '''token_str''': ''' Maul''',
						        },
						        {'''sequence''': '''The largest city in France isELS''', '''score''': 2.2e-0_5, '''token''': 1_6_4_1_6, '''token_str''': '''ELS'''},
						    ]   ,)

						snake_case__	:      Union[str, Any]     		=	unmasker('''My name is <mask>'''   ,targets=[''' Patrick''', ''' Clara''', ''' Teven''']   ,top_k=3  )
						self.assertEqual(
						    nested_simplify(__lowercase   ,decimals=6  )   ,[
						        {'''sequence''': '''My name is Patrick''', '''score''': 2.1e-0_5, '''token''': 3_4_9_9, '''token_str''': ''' Patrick'''},
						        {'''sequence''': '''My name is Te''', '''score''': 2e-0_5, '''token''': 2_9_4_1, '''token_str''': ''' Te'''},
						        {'''sequence''': '''My name is Clara''', '''score''': 2e-0_5, '''token''': 1_3_6_0_6, '''token_str''': ''' Clara'''},
						    ]   ,)

						snake_case__	:      Optional[int]     		=	unmasker('''My name is <mask> <mask>'''   ,top_k=2  )

						self.assertEqual(
						    nested_simplify(__lowercase   ,decimals=6  )   ,[
						        [
						            {
						                '''score''': 2.2e-0_5,
						                '''token''': 3_5_6_7_6,
						                '''token_str''': ''' Maul''',
						                '''sequence''': '''<s>My name is Maul<mask></s>''',
						            },
						            {'''score''': 2.2e-0_5, '''token''': 1_6_4_1_6, '''token_str''': '''ELS''', '''sequence''': '''<s>My name isELS<mask></s>'''},
						        ],
						        [
						            {
						                '''score''': 2.2e-0_5,
						                '''token''': 3_5_6_7_6,
						                '''token_str''': ''' Maul''',
						                '''sequence''': '''<s>My name is<mask> Maul</s>''',
						            },
						            {'''score''': 2.2e-0_5, '''token''': 1_6_4_1_6, '''token_str''': '''ELS''', '''sequence''': '''<s>My name is<mask>ELS</s>'''},
						        ],
						    ]   ,)



		@require_torch_gpu
		def 	__lowerCamelCase			(     self     :int  ):
						snake_case__	:      Optional[int]     		=	pipeline('''fill-mask'''   ,model='''hf-internal-testing/tiny-random-distilbert'''   ,device=0   ,framework='''pt'''  )

						# convert model to fp16
						pipe.model.half()

						snake_case__	:      List[str]     		=	pipe('''Paris is the [MASK] of France.'''  )
						# We actually don't care about the result, we just want to make sure
						# it works, meaning the float16 tensor got casted back to float32
						# for postprocessing.
						self.assertIsInstance(__lowercase   ,__lowercase  )



		@slow
		@require_torch
		def 	__lowerCamelCase			(     self     :str  ):
						snake_case__	:      List[str]     		=	pipeline(task='''fill-mask'''   ,model='''distilroberta-base'''   ,top_k=2   ,framework='''pt'''  )
						self.run_large_test(__lowercase  )



		@slow
		@require_tf
		def 	__lowerCamelCase			(     self     :Any  ):
						snake_case__	:      Optional[Any]     		=	pipeline(task='''fill-mask'''   ,model='''distilroberta-base'''   ,top_k=2   ,framework='''tf'''  )
						self.run_large_test(__lowercase  )



		def 	__lowerCamelCase			(     self     :Optional[Any]   ,__lowercase     :List[Any]  ):
						snake_case__	:      Optional[Any]     		=	unmasker('''My name is <mask>'''  )
						self.assertEqual(
						    nested_simplify(__lowercase  )   ,[
						        {'''sequence''': '''My name is John''', '''score''': 0.008, '''token''': 6_1_0, '''token_str''': ''' John'''},
						        {'''sequence''': '''My name is Chris''', '''score''': 0.007, '''token''': 1_5_7_3, '''token_str''': ''' Chris'''},
						    ]   ,)
						snake_case__	:      str     		=	unmasker('''The largest city in France is <mask>'''  )
						self.assertEqual(
						    nested_simplify(__lowercase  )   ,[
						        {
						            '''sequence''': '''The largest city in France is Paris''',
						            '''score''': 0.251,
						            '''token''': 2_2_0_1,
						            '''token_str''': ''' Paris''',
						        },
						        {
						            '''sequence''': '''The largest city in France is Lyon''',
						            '''score''': 0.214,
						            '''token''': 1_2_7_9_0,
						            '''token_str''': ''' Lyon''',
						        },
						    ]   ,)

						snake_case__	:      Dict     		=	unmasker('''My name is <mask>'''   ,targets=[''' Patrick''', ''' Clara''', ''' Teven''']   ,top_k=3  )
						self.assertEqual(
						    nested_simplify(__lowercase  )   ,[
						        {'''sequence''': '''My name is Patrick''', '''score''': 0.005, '''token''': 3_4_9_9, '''token_str''': ''' Patrick'''},
						        {'''sequence''': '''My name is Clara''', '''score''': 0.000, '''token''': 1_3_6_0_6, '''token_str''': ''' Clara'''},
						        {'''sequence''': '''My name is Te''', '''score''': 0.000, '''token''': 2_9_4_1, '''token_str''': ''' Te'''},
						    ]   ,)



		@require_torch
		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      List[Any]     		=	pipeline(task='''fill-mask'''   ,model='''sshleifer/tiny-distilroberta-base'''   ,framework='''pt'''  )
						snake_case__	:      str     		=	None
						snake_case__	:      int     		=	None
						self.run_pipeline_test(__lowercase   ,[]  )



		@require_tf
		def 	__lowerCamelCase			(     self     :int  ):
						snake_case__	:      Optional[int]     		=	pipeline(task='''fill-mask'''   ,model='''sshleifer/tiny-distilroberta-base'''   ,framework='''tf'''  )
						snake_case__	:      int     		=	None
						snake_case__	:      List[str]     		=	None
						self.run_pipeline_test(__lowercase   ,[]  )



		def 	__lowerCamelCase			(     self     :Any   ,__lowercase     :Any   ,__lowercase     :str   ,__lowercase     :Union[str, Any]  ):
						if tokenizer is None or tokenizer.mask_token_id is None:
										self.skipTest('''The provided tokenizer has no mask token, (probably reformer or wav2vec2)'''  )

						snake_case__	:      Optional[int]     		=	FillMaskPipeline(model=__lowercase   ,tokenizer=__lowercase  )
						snake_case__	:      List[str]     		=	[
						    F"""This is another {tokenizer.mask_token} test""",
						]
						return fill_masker, examples



		def 	__lowerCamelCase			(     self     :Optional[Any]   ,__lowercase     :List[Any]   ,__lowercase     :Optional[Any]  ):
						snake_case__	:      List[str]     		=	fill_masker.tokenizer
						snake_case__	:      List[Any]     		=	fill_masker.model

						snake_case__	:      Dict     		=	fill_masker(
						    F"""This is a {tokenizer.mask_token}"""   ,)
						self.assertEqual(
						    __lowercase   ,[
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						    ]   ,)

						snake_case__	:      Tuple     		=	fill_masker([F"""This is a {tokenizer.mask_token}"""]  )
						self.assertEqual(
						    __lowercase   ,[
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						    ]   ,)

						snake_case__	:      List[str]     		=	fill_masker([F"""This is a {tokenizer.mask_token}""", F"""Another {tokenizer.mask_token} great test."""]  )
						self.assertEqual(
						    __lowercase   ,[
						        [
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        ],
						        [
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        ],
						    ]   ,)

						with self.assertRaises(__lowercase  ):
										fill_masker([None]  )
						# No mask_token is not supported
						with self.assertRaises(__lowercase  ):
										fill_masker('''This is'''  )

						self.run_test_top_k(__lowercase   ,__lowercase  )
						self.run_test_targets(__lowercase   ,__lowercase  )
						self.run_test_top_k_targets(__lowercase   ,__lowercase  )
						self.fill_mask_with_duplicate_targets_and_top_k(__lowercase   ,__lowercase  )
						self.fill_mask_with_multiple_masks(__lowercase   ,__lowercase  )



		def 	__lowerCamelCase			(     self     :Union[str, Any]   ,__lowercase     :Optional[int]   ,__lowercase     :int  ):
						snake_case__	:      int     		=	tokenizer.get_vocab()
						snake_case__	:      Dict     		=	sorted(vocab.keys()  )[:2]
						# Pipeline argument
						snake_case__	:      List[Any]     		=	FillMaskPipeline(model=__lowercase   ,tokenizer=__lowercase   ,targets=__lowercase  )
						snake_case__	:      str     		=	fill_masker(F"""This is a {tokenizer.mask_token}"""  )
						self.assertEqual(
						    __lowercase   ,[
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						    ]   ,)
						snake_case__	:      Optional[Any]     		=	{vocab[el] for el in targets}
						self.assertEqual({el['''token'''] for el in outputs}   ,__lowercase  )
						snake_case__	:      Any     		=	[tokenizer.decode([x]  ) for x in target_ids]
						self.assertEqual({el['''token_str'''] for el in outputs}   ,set(__lowercase  )  )

						# Call argument
						snake_case__	:      str     		=	FillMaskPipeline(model=__lowercase   ,tokenizer=__lowercase  )
						snake_case__	:      int     		=	fill_masker(F"""This is a {tokenizer.mask_token}"""   ,targets=__lowercase  )
						self.assertEqual(
						    __lowercase   ,[
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						    ]   ,)
						snake_case__	:      str     		=	{vocab[el] for el in targets}
						self.assertEqual({el['''token'''] for el in outputs}   ,__lowercase  )
						snake_case__	:      Optional[Any]     		=	[tokenizer.decode([x]  ) for x in target_ids]
						self.assertEqual({el['''token_str'''] for el in outputs}   ,set(__lowercase  )  )

						# Score equivalence
						snake_case__	:      Dict     		=	fill_masker(F"""This is a {tokenizer.mask_token}"""   ,targets=__lowercase  )
						snake_case__	:      Union[str, Any]     		=	[top_mask['''token_str'''] for top_mask in outputs]
						snake_case__	:      Tuple     		=	[top_mask['''score'''] for top_mask in outputs]

						# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
						if set(__lowercase  ) == set(__lowercase  ):
										snake_case__	:      List[Any]     		=	fill_masker(F"""This is a {tokenizer.mask_token}"""   ,targets=__lowercase  )
										snake_case__	:      int     		=	[top_mask['''score'''] for top_mask in unmasked_targets]
										self.assertEqual(nested_simplify(__lowercase  )   ,nested_simplify(__lowercase  )  )

						# Raises with invalid
						with self.assertRaises(__lowercase  ):
										snake_case__	:      List[str]     		=	fill_masker(F"""This is a {tokenizer.mask_token}"""   ,targets=[]  )
						# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
						if "" not in tokenizer.get_vocab():
										with self.assertRaises(__lowercase  ):
														snake_case__	:      List[Any]     		=	fill_masker(F"""This is a {tokenizer.mask_token}"""   ,targets=['''''']  )
										with self.assertRaises(__lowercase  ):
														snake_case__	:      Optional[int]     		=	fill_masker(F"""This is a {tokenizer.mask_token}"""   ,targets=''''''  )



		def 	__lowerCamelCase			(     self     :Any   ,__lowercase     :Union[str, Any]   ,__lowercase     :Dict  ):
						snake_case__	:      int     		=	FillMaskPipeline(model=__lowercase   ,tokenizer=__lowercase   ,top_k=2  )
						snake_case__	:      Tuple     		=	fill_masker(F"""This is a {tokenizer.mask_token}"""  )
						self.assertEqual(
						    __lowercase   ,[
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						    ]   ,)

						snake_case__	:      Any     		=	FillMaskPipeline(model=__lowercase   ,tokenizer=__lowercase  )
						snake_case__	:      Optional[Any]     		=	fill_masker(F"""This is a {tokenizer.mask_token}"""   ,top_k=2  )
						self.assertEqual(
						    __lowercase   ,[
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						    ]   ,)
						self.assertEqual(nested_simplify(__lowercase  )   ,nested_simplify(__lowercase  )  )



		def 	__lowerCamelCase			(     self     :List[Any]   ,__lowercase     :Tuple   ,__lowercase     :str  ):
						snake_case__	:      Optional[int]     		=	tokenizer.get_vocab()
						snake_case__	:      int     		=	FillMaskPipeline(model=__lowercase   ,tokenizer=__lowercase  )

						# top_k=2, ntargets=3
						snake_case__	:      int     		=	sorted(vocab.keys()  )[:3]
						snake_case__	:      List[Any]     		=	fill_masker(F"""This is a {tokenizer.mask_token}"""   ,top_k=2   ,targets=__lowercase  )

						# If we use the most probably targets, and filter differently, we should still
						# have the same results
						snake_case__	:      Dict     		=	[el['''token_str'''] for el in sorted(__lowercase   ,key=lambda __lowercase  : x["score"]   ,reverse=__lowercase  )]
						# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
						if set(__lowercase  ).issubset(__lowercase  ):
										snake_case__	:      List[Any]     		=	fill_masker(F"""This is a {tokenizer.mask_token}"""   ,top_k=3   ,targets=__lowercase  )
										# They should yield exactly the same result
										self.assertEqual(nested_simplify(__lowercase  )   ,nested_simplify(__lowercase  )  )



		def 	__lowerCamelCase			(     self     :Union[str, Any]   ,__lowercase     :Dict   ,__lowercase     :Dict  ):
						snake_case__	:      Union[str, Any]     		=	FillMaskPipeline(model=__lowercase   ,tokenizer=__lowercase  )
						snake_case__	:      str     		=	tokenizer.get_vocab()
						# String duplicates + id duplicates
						snake_case__	:      int     		=	sorted(vocab.keys()  )[:3]
						snake_case__	:      Optional[Any]     		=	[targets[0], targets[1], targets[0], targets[2], targets[1]]
						snake_case__	:      Optional[Any]     		=	fill_masker(F"""My name is {tokenizer.mask_token}"""   ,targets=__lowercase   ,top_k=1_0  )

						# The target list contains duplicates, so we can't output more
						# than them
						self.assertEqual(len(__lowercase  )   ,3  )






		def 	__lowerCamelCase			(     self     :Optional[Any]   ,__lowercase     :List[Any]   ,__lowercase     :Optional[Any]  ):
						snake_case__	:      Any     		=	FillMaskPipeline(model=__lowercase   ,tokenizer=__lowercase  )

						snake_case__	:      Tuple     		=	fill_masker(
						    F"""This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}"""   ,top_k=2  )
						self.assertEqual(
						    __lowercase   ,[
						        [
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        ],
						        [
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        ],
						        [
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						            {'''sequence''': ANY(__lowercase  ), '''score''': ANY(__lowercase  ), '''token''': ANY(__lowercase  ), '''token_str''': ANY(__lowercase  )},
						        ],
						    ]   ,)

