






import gc
import unittest

from transformers import MODEL_FOR_MASKED_LM_MAPPING, TF_MODEL_FOR_MASKED_LM_MAPPING, FillMaskPipeline, pipeline
from transformers.pipelines import PipelineException
from transformers.testing_utils import (
    is_pipeline_test,
    is_torch_available,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_gpu,
    slow,
)

from .test_pipelines_common import ANY







@is_pipeline_test
class 			UpperCAmelCase_						(					unittest.TestCase ):



      """simple docstring"""
      UpperCamelCase_				:					Any           =MODEL_FOR_MASKED_LM_MAPPING
      UpperCamelCase_				:					List[str]           =TF_MODEL_FOR_MASKED_LM_MAPPING
      def     UpperCAmelCase						(				self )				->      Dict:
          super().tearDown()
          # clean-up as much as possible GPU memory occupied by PyTorch
          gc.collect()
          if is_torch_available():
              import torch

              torch.cuda.empty_cache()
      @require_tf
      def     UpperCAmelCase						(				self )				->      Optional[int]:
          UpperCamelCase       :Dict								=   pipeline(task='''fill-mask'''    ,				model='''sshleifer/tiny-distilroberta-base'''    ,				top_k=2    ,				framework='''tf''' )
          UpperCamelCase       :Union[str, Any]								=   unmasker('''My name is <mask>''' )
          self.assertEqual(
              nested_simplify(a_    ,				decimals=6 )    ,				[
                  {'''sequence''': '''My name is grouped''', '''score''': 2.1e-05, '''token''': 3_8015, '''token_str''': ''' grouped'''},
                  {'''sequence''': '''My name is accuser''', '''score''': 2.1e-05, '''token''': 2_5506, '''token_str''': ''' accuser'''},
              ]    ,				)

          UpperCamelCase       :Any								=   unmasker('''The largest city in France is <mask>''' )
          self.assertEqual(
              nested_simplify(a_    ,				decimals=6 )    ,				[
                  {
                      '''sequence''': '''The largest city in France is grouped''',
                      '''score''': 2.1e-05,
                      '''token''': 3_8015,
                      '''token_str''': ''' grouped''',
                  },
                  {
                      '''sequence''': '''The largest city in France is accuser''',
                      '''score''': 2.1e-05,
                      '''token''': 2_5506,
                      '''token_str''': ''' accuser''',
                  },
              ]    ,				)

          UpperCamelCase       :Union[str, Any]								=   unmasker('''My name is <mask>'''    ,				targets=[''' Patrick''', ''' Clara''', ''' Teven''']    ,				top_k=3 )
          self.assertEqual(
              nested_simplify(a_    ,				decimals=6 )    ,				[
                  {'''sequence''': '''My name is Clara''', '''score''': 2e-05, '''token''': 1_3606, '''token_str''': ''' Clara'''},
                  {'''sequence''': '''My name is Patrick''', '''score''': 2e-05, '''token''': 3499, '''token_str''': ''' Patrick'''},
                  {'''sequence''': '''My name is Te''', '''score''': 1.9e-05, '''token''': 2941, '''token_str''': ''' Te'''},
              ]    ,				)
      @require_torch
      def     UpperCAmelCase						(				self )				->      Dict:
          UpperCamelCase       :Optional[int]								=   pipeline(task='''fill-mask'''    ,				model='''sshleifer/tiny-distilroberta-base'''    ,				top_k=2    ,				framework='''pt''' )

          UpperCamelCase       :Any								=   unmasker('''My name is <mask>''' )
          self.assertEqual(
              nested_simplify(a_    ,				decimals=6 )    ,				[
                  {'''sequence''': '''My name is Maul''', '''score''': 2.2e-05, '''token''': 3_5676, '''token_str''': ''' Maul'''},
                  {'''sequence''': '''My name isELS''', '''score''': 2.2e-05, '''token''': 1_6416, '''token_str''': '''ELS'''},
              ]    ,				)

          UpperCamelCase       :Any								=   unmasker('''The largest city in France is <mask>''' )
          self.assertEqual(
              nested_simplify(a_    ,				decimals=6 )    ,				[
                  {
                      '''sequence''': '''The largest city in France is Maul''',
                      '''score''': 2.2e-05,
                      '''token''': 3_5676,
                      '''token_str''': ''' Maul''',
                  },
                  {'''sequence''': '''The largest city in France isELS''', '''score''': 2.2e-05, '''token''': 1_6416, '''token_str''': '''ELS'''},
              ]    ,				)

          UpperCamelCase       :Union[str, Any]								=   unmasker('''My name is <mask>'''    ,				targets=[''' Patrick''', ''' Clara''', ''' Teven''']    ,				top_k=3 )
          self.assertEqual(
              nested_simplify(a_    ,				decimals=6 )    ,				[
                  {'''sequence''': '''My name is Patrick''', '''score''': 2.1e-05, '''token''': 3499, '''token_str''': ''' Patrick'''},
                  {'''sequence''': '''My name is Te''', '''score''': 2e-05, '''token''': 2941, '''token_str''': ''' Te'''},
                  {'''sequence''': '''My name is Clara''', '''score''': 2e-05, '''token''': 1_3606, '''token_str''': ''' Clara'''},
              ]    ,				)

          UpperCamelCase       :List[str]								=   unmasker('''My name is <mask> <mask>'''    ,				top_k=2 )

          self.assertEqual(
              nested_simplify(a_    ,				decimals=6 )    ,				[
                  [
                      {
                          '''score''': 2.2e-05,
                          '''token''': 3_5676,
                          '''token_str''': ''' Maul''',
                          '''sequence''': '''<s>My name is Maul<mask></s>''',
                      },
                      {'''score''': 2.2e-05, '''token''': 1_6416, '''token_str''': '''ELS''', '''sequence''': '''<s>My name isELS<mask></s>'''},
                  ],
                  [
                      {
                          '''score''': 2.2e-05,
                          '''token''': 3_5676,
                          '''token_str''': ''' Maul''',
                          '''sequence''': '''<s>My name is<mask> Maul</s>''',
                      },
                      {'''score''': 2.2e-05, '''token''': 1_6416, '''token_str''': '''ELS''', '''sequence''': '''<s>My name is<mask>ELS</s>'''},
                  ],
              ]    ,				)
      @require_torch_gpu
      def     UpperCAmelCase						(				self )				->      Any:
          UpperCamelCase       :int								=   pipeline('''fill-mask'''    ,				model='''hf-internal-testing/tiny-random-distilbert'''    ,				device=0    ,				framework='''pt''' )

          # convert model to fp16
          pipe.model.half()

          UpperCamelCase       :Dict								=   pipe('''Paris is the [MASK] of France.''' )
          # We actually don't care about the result, we just want to make sure
          # it works, meaning the float16 tensor got casted back to float32
          # for postprocessing.
          self.assertIsInstance(a_    ,				a_ )
      @slow
      @require_torch
      def     UpperCAmelCase						(				self )				->      Optional[int]:
          UpperCamelCase       :Optional[Any]								=   pipeline(task='''fill-mask'''    ,				model='''distilroberta-base'''    ,				top_k=2    ,				framework='''pt''' )
          self.run_large_test(a_ )
      @slow
      @require_tf
      def     UpperCAmelCase						(				self )				->      List[str]:
          UpperCamelCase       :Optional[Any]								=   pipeline(task='''fill-mask'''    ,				model='''distilroberta-base'''    ,				top_k=2    ,				framework='''tf''' )
          self.run_large_test(a_ )
      def     UpperCAmelCase						(				self    ,				SCREAMING_SNAKE_CASE_ )				->      Any:
          UpperCamelCase       :Dict								=   unmasker('''My name is <mask>''' )
          self.assertEqual(
              nested_simplify(a_ )    ,				[
                  {'''sequence''': '''My name is John''', '''score''': 0.008, '''token''': 610, '''token_str''': ''' John'''},
                  {'''sequence''': '''My name is Chris''', '''score''': 0.007, '''token''': 1573, '''token_str''': ''' Chris'''},
              ]    ,				)
          UpperCamelCase       :List[Any]								=   unmasker('''The largest city in France is <mask>''' )
          self.assertEqual(
              nested_simplify(a_ )    ,				[
                  {
                      '''sequence''': '''The largest city in France is Paris''',
                      '''score''': 0.251,
                      '''token''': 2201,
                      '''token_str''': ''' Paris''',
                  },
                  {
                      '''sequence''': '''The largest city in France is Lyon''',
                      '''score''': 0.214,
                      '''token''': 1_2790,
                      '''token_str''': ''' Lyon''',
                  },
              ]    ,				)

          UpperCamelCase       :str								=   unmasker('''My name is <mask>'''    ,				targets=[''' Patrick''', ''' Clara''', ''' Teven''']    ,				top_k=3 )
          self.assertEqual(
              nested_simplify(a_ )    ,				[
                  {'''sequence''': '''My name is Patrick''', '''score''': 0.005, '''token''': 3499, '''token_str''': ''' Patrick'''},
                  {'''sequence''': '''My name is Clara''', '''score''': 0.000, '''token''': 1_3606, '''token_str''': ''' Clara'''},
                  {'''sequence''': '''My name is Te''', '''score''': 0.000, '''token''': 2941, '''token_str''': ''' Te'''},
              ]    ,				)
      @require_torch
      def     UpperCAmelCase						(				self )				->      Dict:
          UpperCamelCase       :Union[str, Any]								=   pipeline(task='''fill-mask'''    ,				model='''sshleifer/tiny-distilroberta-base'''    ,				framework='''pt''' )
          UpperCamelCase       :int								=   None
          UpperCamelCase       :Optional[Any]								=   None
          self.run_pipeline_test(a_    ,				[] )
      @require_tf
      def     UpperCAmelCase						(				self )				->      Optional[int]:
          UpperCamelCase       :str								=   pipeline(task='''fill-mask'''    ,				model='''sshleifer/tiny-distilroberta-base'''    ,				framework='''tf''' )
          UpperCamelCase       :Optional[int]								=   None
          UpperCamelCase       :Optional[Any]								=   None
          self.run_pipeline_test(a_    ,				[] )
      def     UpperCAmelCase						(				self    ,				SCREAMING_SNAKE_CASE_    ,				SCREAMING_SNAKE_CASE_    ,				SCREAMING_SNAKE_CASE_ )				->      Optional[Any]:
          if tokenizer is None or tokenizer.mask_token_id is None:
              self.skipTest('''The provided tokenizer has no mask token, (probably reformer or wav2vec2)''' )

          UpperCamelCase       :str								=   FillMaskPipeline(model=a_    ,				tokenizer=a_ )
          UpperCamelCase       :Tuple								=   [
              F'''This is another {tokenizer.mask_token} test''',
          ]
          return fill_masker, examples
      def     UpperCAmelCase						(				self    ,				SCREAMING_SNAKE_CASE_    ,				SCREAMING_SNAKE_CASE_ )				->      str:
          UpperCamelCase       :Union[str, Any]								=   fill_masker.tokenizer
          UpperCamelCase       :Any								=   fill_masker.model

          UpperCamelCase       :Any								=   fill_masker(
              F'''This is a {tokenizer.mask_token}'''    ,				)
          self.assertEqual(
              a_    ,				[
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
              ]    ,				)

          UpperCamelCase       :Any								=   fill_masker([F'''This is a {tokenizer.mask_token}'''] )
          self.assertEqual(
              a_    ,				[
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
              ]    ,				)

          UpperCamelCase       :Tuple								=   fill_masker([F'''This is a {tokenizer.mask_token}''', F'''Another {tokenizer.mask_token} great test.'''] )
          self.assertEqual(
              a_    ,				[
                  [
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  ],
                  [
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  ],
              ]    ,				)

          with self.assertRaises(a_ ):
              fill_masker([None] )
          # No mask_token is not supported
          with self.assertRaises(a_ ):
              fill_masker('''This is''' )

          self.run_test_top_k(a_    ,				a_ )
          self.run_test_targets(a_    ,				a_ )
          self.run_test_top_k_targets(a_    ,				a_ )
          self.fill_mask_with_duplicate_targets_and_top_k(a_    ,				a_ )
          self.fill_mask_with_multiple_masks(a_    ,				a_ )
      def     UpperCAmelCase						(				self    ,				SCREAMING_SNAKE_CASE_    ,				SCREAMING_SNAKE_CASE_ )				->      Optional[int]:
          UpperCamelCase       :str								=   tokenizer.get_vocab()
          UpperCamelCase       :Union[str, Any]								=   sorted(vocab.keys() )[:2]
          # Pipeline argument
          UpperCamelCase       :Dict								=   FillMaskPipeline(model=a_    ,				tokenizer=a_    ,				targets=a_ )
          UpperCamelCase       :str								=   fill_masker(F'''This is a {tokenizer.mask_token}''' )
          self.assertEqual(
              a_    ,				[
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
              ]    ,				)
          UpperCamelCase       :str								=   {vocab[el] for el in targets}
          self.assertEqual({el['''token'''] for el in outputs}    ,				a_ )
          UpperCamelCase       :List[str]								=   [tokenizer.decode([x] ) for x in target_ids]
          self.assertEqual({el['''token_str'''] for el in outputs}    ,				set(a_ ) )

          # Call argument
          UpperCamelCase       :Optional[int]								=   FillMaskPipeline(model=a_    ,				tokenizer=a_ )
          UpperCamelCase       :Optional[Any]								=   fill_masker(F'''This is a {tokenizer.mask_token}'''    ,				targets=a_ )
          self.assertEqual(
              a_    ,				[
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
              ]    ,				)
          UpperCamelCase       :Optional[Any]								=   {vocab[el] for el in targets}
          self.assertEqual({el['''token'''] for el in outputs}    ,				a_ )
          UpperCamelCase       :Optional[int]								=   [tokenizer.decode([x] ) for x in target_ids]
          self.assertEqual({el['''token_str'''] for el in outputs}    ,				set(a_ ) )

          # Score equivalence
          UpperCamelCase       :Optional[int]								=   fill_masker(F'''This is a {tokenizer.mask_token}'''    ,				targets=a_ )
          UpperCamelCase       :Optional[int]								=   [top_mask["""token_str"""] for top_mask in outputs]
          UpperCamelCase       :Optional[int]								=   [top_mask["""score"""] for top_mask in outputs]

          # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
          if set(a_ ) == set(a_ ):
              UpperCamelCase       :str								=   fill_masker(F'''This is a {tokenizer.mask_token}'''    ,				targets=a_ )
              UpperCamelCase       :Optional[int]								=   [top_mask["""score"""] for top_mask in unmasked_targets]
              self.assertEqual(nested_simplify(a_ )    ,				nested_simplify(a_ ) )

          # Raises with invalid
          with self.assertRaises(a_ ):
              UpperCamelCase       :List[str]								=   fill_masker(F'''This is a {tokenizer.mask_token}'''    ,				targets=[] )
          # For some tokenizers, `""` is actually in the vocabulary and the expected error won't raised
          if "" not in tokenizer.get_vocab():
              with self.assertRaises(a_ ):
                  UpperCamelCase       :List[str]								=   fill_masker(F'''This is a {tokenizer.mask_token}'''    ,				targets=[''''''] )
              with self.assertRaises(a_ ):
                  UpperCamelCase       :Any								=   fill_masker(F'''This is a {tokenizer.mask_token}'''    ,				targets='''''' )
      def     UpperCAmelCase						(				self    ,				SCREAMING_SNAKE_CASE_    ,				SCREAMING_SNAKE_CASE_ )				->      Any:
          UpperCamelCase       :List[str]								=   FillMaskPipeline(model=a_    ,				tokenizer=a_    ,				top_k=2 )
          UpperCamelCase       :str								=   fill_masker(F'''This is a {tokenizer.mask_token}''' )
          self.assertEqual(
              a_    ,				[
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
              ]    ,				)

          UpperCamelCase       :Union[str, Any]								=   FillMaskPipeline(model=a_    ,				tokenizer=a_ )
          UpperCamelCase       :int								=   fill_masker(F'''This is a {tokenizer.mask_token}'''    ,				top_k=2 )
          self.assertEqual(
              a_    ,				[
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
              ]    ,				)
          self.assertEqual(nested_simplify(a_ )    ,				nested_simplify(a_ ) )
      def     UpperCAmelCase						(				self    ,				SCREAMING_SNAKE_CASE_    ,				SCREAMING_SNAKE_CASE_ )				->      Optional[int]:
          UpperCamelCase       :Union[str, Any]								=   tokenizer.get_vocab()
          UpperCamelCase       :List[Any]								=   FillMaskPipeline(model=a_    ,				tokenizer=a_ )

          # top_k=2, ntargets=3
          UpperCamelCase       :Tuple								=   sorted(vocab.keys() )[:3]
          UpperCamelCase       :Dict								=   fill_masker(F'''This is a {tokenizer.mask_token}'''    ,				top_k=2    ,				targets=a_ )

          # If we use the most probably targets, and filter differently, we should still
          # have the same results
          UpperCamelCase       :Any								=   [el["""token_str"""] for el in sorted(a_    ,				key=lambda SCREAMING_SNAKE_CASE_ : x["score"]    ,				reverse=a_ )]
          # For some BPE tokenizers, `</w>` is removed during decoding, so `token_str` won't be the same as in `targets`.
          if set(a_ ).issubset(a_ ):
              UpperCamelCase       :Any								=   fill_masker(F'''This is a {tokenizer.mask_token}'''    ,				top_k=3    ,				targets=a_ )
              # They should yield exactly the same result
              self.assertEqual(nested_simplify(a_ )    ,				nested_simplify(a_ ) )
      def     UpperCAmelCase						(				self    ,				SCREAMING_SNAKE_CASE_    ,				SCREAMING_SNAKE_CASE_ )				->      List[Any]:
          UpperCamelCase       :Optional[Any]								=   FillMaskPipeline(model=a_    ,				tokenizer=a_ )
          UpperCamelCase       :Any								=   tokenizer.get_vocab()
          # String duplicates + id duplicates
          UpperCamelCase       :Optional[Any]								=   sorted(vocab.keys() )[:3]
          UpperCamelCase       :Optional[Any]								=   [targets[0], targets[1], targets[0], targets[2], targets[1]]
          UpperCamelCase       :str								=   fill_masker(F'''My name is {tokenizer.mask_token}'''    ,				targets=a_    ,				top_k=10 )

          # The target list contains duplicates, so we can't output more
          # than them
          self.assertEqual(len(a_ )    ,				3 )




      def     UpperCAmelCase						(				self    ,				SCREAMING_SNAKE_CASE_    ,				SCREAMING_SNAKE_CASE_ )				->      Union[str, Any]:
          UpperCamelCase       :Optional[int]								=   FillMaskPipeline(model=a_    ,				tokenizer=a_ )

          UpperCamelCase       :str								=   fill_masker(
              F'''This is a {tokenizer.mask_token} {tokenizer.mask_token} {tokenizer.mask_token}'''    ,				top_k=2 )
          self.assertEqual(
              a_    ,				[
                  [
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  ],
                  [
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  ],
                  [
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                      {'''sequence''': ANY(a_ ), '''score''': ANY(a_ ), '''token''': ANY(a_ ), '''token_str''': ANY(a_ )},
                  ],
              ]    ,				)










'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import _LazyModule


A_							: int						 =	{"""tokenization_bertweet""": ["""BertweetTokenizer"""]}


if TYPE_CHECKING:
   from .tokenization_bertweet import BertweetTokenizer

else:
   import sys

   A_							: int						 =	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



