






import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY



@is_pipeline_test
class       __UpperCAmelCase		(unittest.TestCase    ):
			__snake_case				:      Union[str, Any] 						=							MODEL_FOR_MASKED_LM_MAPPING
			__snake_case				:      List[str] 						=							TF_MODEL_FOR_MASKED_LM_MAPPING







			def 							UpperCamelCase      (	self:  Optional[int]     ):







									'''simple docstring'''




									super().tearDown()
									# clean-up as much as possible GPU memory occupied by PyTorch
									gc.collect()
									if is_torch_available():
															import torch

															torch.cuda.empty_cache()







			@require_tf
			def 							UpperCamelCase      (	self:  Optional[Any]     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  pipeline(task="""fill-mask"""      ,	model="""sshleifer/tiny-distilroberta-base"""      ,	top_k=2      ,	framework="""tf"""     )
									_SCREAMING_SNAKE_CASE	      =  unmasker("""My name is <mask>"""     )
									self.assertEqual(
									    nested_simplify(UpperCAmelCase_      ,	decimals=6     )      ,	[
									        {"""sequence""": """My name is grouped""", """score""": 2.1E-05, """token""": 38_015, """token_str""": """ grouped"""},
									        {"""sequence""": """My name is accuser""", """score""": 2.1E-05, """token""": 25_506, """token_str""": """ accuser"""},
									    ]      ,	)

									_SCREAMING_SNAKE_CASE	      =  unmasker("""The largest city in France is <mask>"""     )
									self.assertEqual(
									    nested_simplify(UpperCAmelCase_      ,	decimals=6     )      ,	[
									        {
									            """sequence""": """The largest city in France is grouped""",
									            """score""": 2.1E-05,
									            """token""": 38_015,
									            """token_str""": """ grouped""",
									        },
									        {
									            """sequence""": """The largest city in France is accuser""",
									            """score""": 2.1E-05,
									            """token""": 25_506,
									            """token_str""": """ accuser""",
									        },
									    ]      ,	)

									_SCREAMING_SNAKE_CASE	      =  unmasker("""My name is <mask>"""      ,	targets=[""" Patrick""", """ Clara""", """ Teven"""]      ,	top_k=3     )
									self.assertEqual(
									    nested_simplify(UpperCAmelCase_      ,	decimals=6     )      ,	[
									        {"""sequence""": """My name is Clara""", """score""": 2E-05, """token""": 13_606, """token_str""": """ Clara"""},
									        {"""sequence""": """My name is Patrick""", """score""": 2E-05, """token""": 3_499, """token_str""": """ Patrick"""},
									        {"""sequence""": """My name is Te""", """score""": 1.9E-05, """token""": 2_941, """token_str""": """ Te"""},
									    ]      ,	)







			@require_torch
			def 							UpperCamelCase      (	self:  int     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  pipeline(task="""fill-mask"""      ,	model="""sshleifer/tiny-distilroberta-base"""      ,	top_k=2      ,	framework="""pt"""     )

									_SCREAMING_SNAKE_CASE	      =  unmasker("""My name is <mask>"""     )
									self.assertEqual(
									    nested_simplify(UpperCAmelCase_      ,	decimals=6     )      ,	[
									        {"""sequence""": """My name is Maul""", """score""": 2.2E-05, """token""": 35_676, """token_str""": """ Maul"""},
									        {"""sequence""": """My name isELS""", """score""": 2.2E-05, """token""": 16_416, """token_str""": """ELS"""},
									    ]      ,	)

									_SCREAMING_SNAKE_CASE	      =  unmasker("""The largest city in France is <mask>"""     )
									self.assertEqual(
									    nested_simplify(UpperCAmelCase_      ,	decimals=6     )      ,	[
									        {
									            """sequence""": """The largest city in France is Maul""",
									            """score""": 2.2E-05,
									            """token""": 35_676,
									            """token_str""": """ Maul""",
									        },
									        {"""sequence""": """The largest city in France isELS""", """score""": 2.2E-05, """token""": 16_416, """token_str""": """ELS"""},
									    ]      ,	)

									_SCREAMING_SNAKE_CASE	      =  unmasker("""My name is <mask>"""      ,	targets=[""" Patrick""", """ Clara""", """ Teven"""]      ,	top_k=3     )
									self.assertEqual(
									    nested_simplify(UpperCAmelCase_      ,	decimals=6     )      ,	[
									        {"""sequence""": """My name is Patrick""", """score""": 2.1E-05, """token""": 3_499, """token_str""": """ Patrick"""},
									        {"""sequence""": """My name is Te""", """score""": 2E-05, """token""": 2_941, """token_str""": """ Te"""},
									        {"""sequence""": """My name is Clara""", """score""": 2E-05, """token""": 13_606, """token_str""": """ Clara"""},
									    ]      ,	)

									_SCREAMING_SNAKE_CASE	      =  unmasker("""My name is <mask> <mask>"""      ,	top_k=2     )

									self.assertEqual(
									    nested_simplify(UpperCAmelCase_      ,	decimals=6     )      ,	[
									        [
									            {
									                """score""": 2.2E-05,
									                """token""": 35_676,
									                """token_str""": """ Maul""",
									                """sequence""": """<s>My name is Maul<mask></s>""",
									            },
									            {"""score""": 2.2E-05, """token""": 16_416, """token_str""": """ELS""", """sequence""": """<s>My name isELS<mask></s>"""},
									        ],
									        [
									            {
									                """score""": 2.2E-05,
									                """token""": 35_676,
									                """token_str""": """ Maul""",
									                """sequence""": """<s>My name is<mask> Maul</s>""",
									            },
									            {"""score""": 2.2E-05, """token""": 16_416, """token_str""": """ELS""", """sequence""": """<s>My name is<mask>ELS</s>"""},
									        ],
									    ]      ,	)







			@require_torch_gpu
			def 							UpperCamelCase      (	self:  Tuple     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  pipeline("""fill-mask"""      ,	model="""hf-internal-testing/tiny-random-distilbert"""      ,	device=0      ,	framework="""pt"""     )

									# convert model to fp16
									pipe.model.half()

									_SCREAMING_SNAKE_CASE	      =  pipe("""Paris is the [MASK] of France."""     )
									# We actually don't care about the result, we just want to make sure
									# it works, meaning the float16 tensor got casted back to float32
									# for postprocessing.
									self.assertIsInstance(UpperCAmelCase_      ,	UpperCAmelCase_     )







			@slow
			@require_torch
			def 							UpperCamelCase      (	self:  Any     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  pipeline(task="""fill-mask"""      ,	model="""distilroberta-base"""      ,	top_k=2      ,	framework="""pt"""     )
									self.run_large_test(UpperCAmelCase_     )







			@slow
			@require_tf
			def 							UpperCamelCase      (	self:  Union[str, Any]     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  pipeline(task="""fill-mask"""      ,	model="""distilroberta-base"""      ,	top_k=2      ,	framework="""tf"""     )
									self.run_large_test(UpperCAmelCase_     )







			def 							UpperCamelCase      (	self:  Union[str, Any]      ,	UpperCAmelCase_:  Optional[int]     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  unmasker("""My name is <mask>"""     )
									self.assertEqual(
									    nested_simplify(UpperCAmelCase_     )      ,	[
									        {"""sequence""": """My name is John""", """score""": 0.0_08, """token""": 610, """token_str""": """ John"""},
									        {"""sequence""": """My name is Chris""", """score""": 0.0_07, """token""": 1_573, """token_str""": """ Chris"""},
									    ]      ,	)
									_SCREAMING_SNAKE_CASE	      =  unmasker("""The largest city in France is <mask>"""     )
									self.assertEqual(
									    nested_simplify(UpperCAmelCase_     )      ,	[
									        {
									            """sequence""": """The largest city in France is Paris""",
									            """score""": 0.2_51,
									            """token""": 2_201,
									            """token_str""": """ Paris""",
									        },
									        {
									            """sequence""": """The largest city in France is Lyon""",
									            """score""": 0.2_14,
									            """token""": 12_790,
									            """token_str""": """ Lyon""",
									        },
									    ]      ,	)

									_SCREAMING_SNAKE_CASE	      =  unmasker("""My name is <mask>"""      ,	targets=[""" Patrick""", """ Clara""", """ Teven"""]      ,	top_k=3     )
									self.assertEqual(
									    nested_simplify(UpperCAmelCase_     )      ,	[
									        {"""sequence""": """My name is Patrick""", """score""": 0.0_05, """token""": 3_499, """token_str""": """ Patrick"""},
									        {"""sequence""": """My name is Clara""", """score""": 0.0_00, """token""": 13_606, """token_str""": """ Clara"""},
									        {"""sequence""": """My name is Te""", """score""": 0.0_00, """token""": 2_941, """token_str""": """ Te"""},
									    ]      ,	)







			@require_torch
			def 							UpperCamelCase      (	self:  Any     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  pipeline(task="""fill-mask"""      ,	model="""sshleifer/tiny-distilroberta-base"""      ,	framework="""pt"""     )
									_SCREAMING_SNAKE_CASE	      =  None
									_SCREAMING_SNAKE_CASE	      =  None
									self.run_pipeline_test(UpperCAmelCase_      ,	[]     )







			@require_tf
			def 							UpperCamelCase      (	self:  str     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  pipeline(task="""fill-mask"""      ,	model="""sshleifer/tiny-distilroberta-base"""      ,	framework="""tf"""     )
									_SCREAMING_SNAKE_CASE	      =  None
									_SCREAMING_SNAKE_CASE	      =  None
									self.run_pipeline_test(UpperCAmelCase_      ,	[]     )







			def 							UpperCamelCase      (	self:  Tuple      ,	UpperCAmelCase_:  Optional[Any]      ,	UpperCAmelCase_:  int      ,	UpperCAmelCase_:  Optional[int]     ):







									'''simple docstring'''




									if tokenizer is None or tokenizer.mask_token_id is None:
															self.skipTest("""The provided tokenizer has no mask token, (probably reformer or wav2vec2)"""     )

									_SCREAMING_SNAKE_CASE	      =  FillMaskPipeline(model=UpperCAmelCase_      ,	tokenizer=UpperCAmelCase_     )
									_SCREAMING_SNAKE_CASE	      =  [
									    F'This is another {tokenizer.mask_token} test',
									]
									return fill_masker, examples







			def 							UpperCamelCase      (	self:  Union[str, Any]      ,	UpperCAmelCase_:  Optional[int]      ,	UpperCAmelCase_:  Any     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  fill_masker.tokenizer
									_SCREAMING_SNAKE_CASE	      =  fill_masker.model

									_SCREAMING_SNAKE_CASE	      =  fill_masker(
									    F'This is a {tokenizer.mask_token}'      ,	)
									self.assertEqual(
									    UpperCAmelCase_      ,	[
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									    ]      ,	)

									_SCREAMING_SNAKE_CASE	      =  fill_masker([F'This is a {tokenizer.mask_token}']     )
									self.assertEqual(
									    UpperCAmelCase_      ,	[
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									    ]      ,	)

									_SCREAMING_SNAKE_CASE	      =  fill_masker([F'This is a {tokenizer.mask_token}', F'Another {tokenizer.mask_token} great test.']     )
									self.assertEqual(
									    UpperCAmelCase_      ,	[
									        [
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        ],
									        [
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        ],
									    ]      ,	)

									with self.assertRaises(UpperCAmelCase_     ):
															fill_masker([None]     )
									# No mask_token is not supported
									with self.assertRaises(UpperCAmelCase_     ):
															fill_masker("""This is"""     )

									self.run_test_top_k(UpperCAmelCase_      ,	UpperCAmelCase_     )
									self.run_test_targets(UpperCAmelCase_      ,	UpperCAmelCase_     )
									self.run_test_top_k_targets(UpperCAmelCase_      ,	UpperCAmelCase_     )
									self.fill_mask_with_duplicate_targets_and_top_k(UpperCAmelCase_      ,	UpperCAmelCase_     )
									self.fill_mask_with_multiple_masks(UpperCAmelCase_      ,	UpperCAmelCase_     )







			def 							UpperCamelCase      (	self:  Dict      ,	UpperCAmelCase_:  Tuple      ,	UpperCAmelCase_:  int     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  tokenizer.get_vocab()
									_SCREAMING_SNAKE_CASE	      =  sorted(vocab.keys()     )[:2]
									# Pipeline argument
									_SCREAMING_SNAKE_CASE	      =  FillMaskPipeline(model=UpperCAmelCase_      ,	tokenizer=UpperCAmelCase_      ,	targets=UpperCAmelCase_     )
									_SCREAMING_SNAKE_CASE	      =  fill_masker(F'This is a {tokenizer.mask_token}'     )
									self.assertEqual(
									    UpperCAmelCase_      ,	[
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									    ]      ,	)
									_SCREAMING_SNAKE_CASE	      =  {vocab[el] for el in targets}
									self.assertEqual({el["""token"""] for el in outputs}      ,	UpperCAmelCase_     )
									_SCREAMING_SNAKE_CASE	      =  [tokenizer.decode([x]     ) for x in target_ids]
									self.assertEqual({el["""token_str"""] for el in outputs}      ,	set(UpperCAmelCase_     )     )

									# Call argument
									_SCREAMING_SNAKE_CASE	      =  FillMaskPipeline(model=UpperCAmelCase_      ,	tokenizer=UpperCAmelCase_     )
									_SCREAMING_SNAKE_CASE	      =  fill_masker(F'This is a {tokenizer.mask_token}'      ,	targets=UpperCAmelCase_     )
									self.assertEqual(
									    UpperCAmelCase_      ,	[
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									    ]      ,	)
									_SCREAMING_SNAKE_CASE	      =  {vocab[el] for el in targets}
									self.assertEqual({el["""token"""] for el in outputs}      ,	UpperCAmelCase_     )
									_SCREAMING_SNAKE_CASE	      =  [tokenizer.decode([x]     ) for x in target_ids]
									self.assertEqual({el["""token_str"""] for el in outputs}      ,	set(UpperCAmelCase_     )     )

									# Score equivalence
									_SCREAMING_SNAKE_CASE	      =  fill_masker(F'This is a {tokenizer.mask_token}'      ,	targets=UpperCAmelCase_     )
									_SCREAMING_SNAKE_CASE	      =  [top_mask["""token_str"""] for top_mask in outputs]
									_SCREAMING_SNAKE_CASE	      =  [top_mask["""score"""] for top_mask in outputs]

									# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
									if set(UpperCAmelCase_     ) == set(UpperCAmelCase_     ):
															_SCREAMING_SNAKE_CASE	      =  fill_masker(F'This is a {tokenizer.mask_token}'      ,	targets=UpperCAmelCase_     )
															_SCREAMING_SNAKE_CASE	      =  [top_mask["""score"""] for top_mask in unmasked_targets]
															self.assertEqual(nested_simplify(UpperCAmelCase_     )      ,	nested_simplify(UpperCAmelCase_     )     )

									# Raises with invalid
									with self.assertRaises(UpperCAmelCase_     ):
															_SCREAMING_SNAKE_CASE	      =  fill_masker(F'This is a {tokenizer.mask_token}'      ,	targets=[]     )
									# For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
									if "" not in tokenizer.get_vocab():
															with self.assertRaises(UpperCAmelCase_     ):
																					_SCREAMING_SNAKE_CASE	      =  fill_masker(F'This is a {tokenizer.mask_token}'      ,	targets=[""""""]     )
															with self.assertRaises(UpperCAmelCase_     ):
																					_SCREAMING_SNAKE_CASE	      =  fill_masker(F'This is a {tokenizer.mask_token}'      ,	targets=""""""     )







			def 							UpperCamelCase      (	self:  Optional[int]      ,	UpperCAmelCase_:  Optional[int]      ,	UpperCAmelCase_:  int     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  FillMaskPipeline(model=UpperCAmelCase_      ,	tokenizer=UpperCAmelCase_      ,	top_k=2     )
									_SCREAMING_SNAKE_CASE	      =  fill_masker(F'This is a {tokenizer.mask_token}'     )
									self.assertEqual(
									    UpperCAmelCase_      ,	[
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									    ]      ,	)

									_SCREAMING_SNAKE_CASE	      =  FillMaskPipeline(model=UpperCAmelCase_      ,	tokenizer=UpperCAmelCase_     )
									_SCREAMING_SNAKE_CASE	      =  fill_masker(F'This is a {tokenizer.mask_token}'      ,	top_k=2     )
									self.assertEqual(
									    UpperCAmelCase_      ,	[
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									    ]      ,	)
									self.assertEqual(nested_simplify(UpperCAmelCase_     )      ,	nested_simplify(UpperCAmelCase_     )     )







			def 							UpperCamelCase      (	self:  List[str]      ,	UpperCAmelCase_:  Tuple      ,	UpperCAmelCase_:  List[Any]     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  tokenizer.get_vocab()
									_SCREAMING_SNAKE_CASE	      =  FillMaskPipeline(model=UpperCAmelCase_      ,	tokenizer=UpperCAmelCase_     )

									# top_k=2, ntargets=3
									_SCREAMING_SNAKE_CASE	      =  sorted(vocab.keys()     )[:3]
									_SCREAMING_SNAKE_CASE	      =  fill_masker(F'This is a {tokenizer.mask_token}'      ,	top_k=2      ,	targets=UpperCAmelCase_     )

									# If we use the most probably targets, and filter differently, we should still
									# have the same results
									_SCREAMING_SNAKE_CASE	      =  [el["""token_str"""] for el in sorted(UpperCAmelCase_      ,	key=lambda UpperCAmelCase_     : x["score"]      ,	reverse=UpperCAmelCase_     )]
									# For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
									if set(UpperCAmelCase_     ).issubset(UpperCAmelCase_     ):
															_SCREAMING_SNAKE_CASE	      =  fill_masker(F'This is a {tokenizer.mask_token}'      ,	top_k=3      ,	targets=UpperCAmelCase_     )
															# They should yield exactly the same result
															self.assertEqual(nested_simplify(UpperCAmelCase_     )      ,	nested_simplify(UpperCAmelCase_     )     )







			def 							UpperCamelCase      (	self:  Union[str, Any]      ,	UpperCAmelCase_:  int      ,	UpperCAmelCase_:  List[str]     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  FillMaskPipeline(model=UpperCAmelCase_      ,	tokenizer=UpperCAmelCase_     )
									_SCREAMING_SNAKE_CASE	      =  tokenizer.get_vocab()
									# String duplicates + id duplicates
									_SCREAMING_SNAKE_CASE	      =  sorted(vocab.keys()     )[:3]
									_SCREAMING_SNAKE_CASE	      =  [targets[0], targets[1], targets[0], targets[2], targets[1]]
									_SCREAMING_SNAKE_CASE	      =  fill_masker(F'My name is {tokenizer.mask_token}'      ,	targets=UpperCAmelCase_      ,	top_k=10     )

									# The target list contains duplicates, so we can't output more
									# than them
									self.assertEqual(len(UpperCAmelCase_     )      ,	3     )







			def 							UpperCamelCase      (	self:  List[str]      ,	UpperCAmelCase_:  Optional[int]      ,	UpperCAmelCase_:  str     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE	      =  FillMaskPipeline(model=UpperCAmelCase_      ,	tokenizer=UpperCAmelCase_     )

									_SCREAMING_SNAKE_CASE	      =  fill_masker(
									    F'This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}'      ,	top_k=2     )
									self.assertEqual(
									    UpperCAmelCase_      ,	[
									        [
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        ],
									        [
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        ],
									        [
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									            {"""sequence""": ANY(UpperCAmelCase_     ), """score""": ANY(UpperCAmelCase_     ), """token""": ANY(UpperCAmelCase_     ), """token_str""": ANY(UpperCAmelCase_     )},
									        ],
									    ]      ,	)









from typing import List, Optional, Union

from ...image_utils import ImageInput
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType




class  UpperCamelCase_      (			snake_case_				):



	'''simple docstring'''




	lowerCAmelCase													= ['''image_processor''', '''tokenizer''']
	lowerCAmelCase													= '''BlipImageProcessor'''
	lowerCAmelCase													= ('''BertTokenizer''', '''BertTokenizerFast''')




	def __init__(       self       ,			a       ,			a					)       ->    Tuple:
								snake_case_				=							False
								super().__init__(a       ,			a					)
								snake_case_				=							self.image_processor




	def __call__(       self       ,			a = None       ,			a = None       ,			a = True       ,			a = False       ,			a = None       ,			a = None       ,			a = 0       ,			a = None       ,			a = None       ,			a = False       ,			a = False       ,			a = False       ,			a = False       ,			a = False       ,			a = True       ,			a = None       ,			**a       ,			)       ->    BatchEncoding:
								if images is None and text is None:
															raise ValueError('You have to specify either images or text.'					)

								# Get only text
								if images is None:
															snake_case_				=							self.tokenizer
															snake_case_				=							self.tokenizer(
															    text=a       ,			add_special_tokens=a       ,			padding=a       ,			truncation=a       ,			max_length=a       ,			stride=a       ,			pad_to_multiple_of=a       ,			return_attention_mask=a       ,			return_overflowing_tokens=a       ,			return_special_tokens_mask=a       ,			return_offsets_mapping=a       ,			return_token_type_ids=a       ,			return_length=a       ,			verbose=a       ,			return_tensors=a       ,			**a       ,			)
															return text_encoding

								# add pixel_values
								snake_case_				=							self.image_processor(a       ,			return_tensors=a					)

								if text is not None:
															snake_case_				=							self.tokenizer(
															    text=a       ,			add_special_tokens=a       ,			padding=a       ,			truncation=a       ,			max_length=a       ,			stride=a       ,			pad_to_multiple_of=a       ,			return_attention_mask=a       ,			return_overflowing_tokens=a       ,			return_special_tokens_mask=a       ,			return_offsets_mapping=a       ,			return_token_type_ids=a       ,			return_length=a       ,			verbose=a       ,			return_tensors=a       ,			**a       ,			)
								else:
															snake_case_				=							None

								if text_encoding is not None:
															encoding_image_processor.update(a					)

								return encoding_image_processor




	def 							_UpperCamelCase			(       self       ,			*a       ,			**a					)       ->    int:
								return self.tokenizer.batch_decode(*a       ,			**a					)




	def 							_UpperCamelCase			(       self       ,			*a       ,			**a					)       ->    Any:
								return self.tokenizer.decode(*a       ,			**a					)






	@property
	def 							_UpperCamelCase			(       self					)       ->    List[str]:
								snake_case_				=							self.tokenizer.model_input_names
								snake_case_				=							self.image_processor.model_input_names
								return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names					)					)





