





'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


A :			List[str]							   =      logging.get_logger(__name__)

A :			List[Any]							   =      {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

A :			Any							   =      {
    'vocab_file': {
        'microsoft/mpnet-base': 'https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'microsoft/mpnet-base': 'https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json',
    },
}

A :			Dict							   =      {
    'microsoft/mpnet-base': 512,
}

A :			Optional[int]							   =      {
    'microsoft/mpnet-base': {'do_lower_case': True},
}







class       lowerCamelCase       (    __UpperCAmelCase	):
					_SCREAMING_SNAKE_CASE				 =     VOCAB_FILES_NAMES
					_SCREAMING_SNAKE_CASE				 =     PRETRAINED_VOCAB_FILES_MAP
					_SCREAMING_SNAKE_CASE				 =     PRETRAINED_INIT_CONFIGURATION
					_SCREAMING_SNAKE_CASE				 =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_SCREAMING_SNAKE_CASE				 =     MPNetTokenizer
					_SCREAMING_SNAKE_CASE				 =     ["input_ids", "attention_mask"]


					def __init__(       self						:	int							,  __snake_case						:	Optional[Any]=None							,  __snake_case						:	Any=None							,  __snake_case						:	Any=True							,  __snake_case						:	List[str]="<s>"							,  __snake_case						:	int="</s>"							,  __snake_case						:	Dict="</s>"							,  __snake_case						:	int="<s>"							,  __snake_case						:	Dict="[UNK]"							,  __snake_case						:	Optional[int]="<pad>"							,  __snake_case						:	Optional[Any]="<mask>"							,  __snake_case						:	Any=True							,  __snake_case						:	Any=None							,  **__snake_case						:	Optional[int]							,  ):





											'''simple docstring'''





											super().__init__(
											    __snake_case							,  tokenizer_file=__snake_case							,  do_lower_case=__snake_case							,  bos_token=__snake_case							,  eos_token=__snake_case							,  sep_token=__snake_case							,  cls_token=__snake_case							,  unk_token=__snake_case							,  pad_token=__snake_case							,  mask_token=__snake_case							,  tokenize_chinese_chars=__snake_case							,  strip_accents=__snake_case							,  **__snake_case							,  )

											_snake_case: Dict	       =		json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
											if (
											    pre_tok_state.get('lowercase'							,  __snake_case							) != do_lower_case
											    or pre_tok_state.get('strip_accents'							,  __snake_case							) != strip_accents
											):
																	_snake_case: List[str]	       =		getattr(__snake_case							,  pre_tok_state.pop('type'							)							)
																	_snake_case: Optional[Any]	       =		do_lower_case
																	_snake_case: Tuple	       =		strip_accents
																	_snake_case: List[str]	       =		pre_tok_class(**__snake_case							)

											_snake_case: Any	       =		do_lower_case


					@property
					def  SCREAMING_SNAKE_CASE_		(       self						:	List[str]							):





											'''simple docstring'''





											if self._mask_token is None:
																	if self.verbose:
																							logger.error('Using mask_token, but it is not set yet.'							)
																	return None
											return str(self._mask_token							)


					@mask_token.setter
					def  SCREAMING_SNAKE_CASE_		(       self						:	List[Any]							,  __snake_case						:	str							):





											'''simple docstring'''





											_snake_case: Union[str, Any]	       =		AddedToken(__snake_case							,  lstrip=__snake_case							,  rstrip=__snake_case							) if isinstance(__snake_case							,  __snake_case							) else value
											_snake_case: Dict	       =		value


					def  SCREAMING_SNAKE_CASE_		(       self						:	List[str]							,  __snake_case						:	Any							,  __snake_case						:	Union[str, Any]=None							):





											'''simple docstring'''





											_snake_case: Any	       =		[self.bos_token_id] + token_ids_a + [self.eos_token_id]
											if token_ids_a is None:
																	return output

											return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]


					def  SCREAMING_SNAKE_CASE_		(       self						:	Union[str, Any]							,  __snake_case						:	List[int]							,  __snake_case						:	Optional[List[int]] = None							):





											'''simple docstring'''





											_snake_case: List[Any]	       =		[self.sep_token_id]
											_snake_case: Dict	       =		[self.cls_token_id]

											if token_ids_a is None:
																	return len(cls + token_ids_a + sep							) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]




					def  SCREAMING_SNAKE_CASE_		(       self						:	str							,  __snake_case						:	str							,  __snake_case						:	Optional[str] = None							):





											'''simple docstring'''





											_snake_case: Any	       =		self._tokenizer.model.save(__snake_case							,  name=__snake_case							)
											return tuple(__snake_case							)












'''simple docstring'''






import unittest

from transformers import (
    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
    PreTrainedTokenizer,
    is_vision_available,
)
from transformers.pipelines import ImageClassificationPipeline, pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_tf,
    require_torch,
    require_torch_or_tf,
    require_vision,
    slow,
)

from .test_pipelines_common import ANY


if is_vision_available():
		from PIL import Image
else:







		class       lowerCamelCase       :


							@staticmethod
							def  SCREAMING_SNAKE_CASE_		(       *__snake_case						:	Optional[Any]							,  **__snake_case						:	str							):





													'''simple docstring'''





													pass







@is_pipeline_test
@require_torch_or_tf
@require_vision
class       lowerCamelCase       (    unittest.TestCase	):
					_SCREAMING_SNAKE_CASE				 =     MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
					_SCREAMING_SNAKE_CASE				 =     TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING


					def  SCREAMING_SNAKE_CASE_		(       self						:	Union[str, Any]							,  __snake_case						:	Any							,  __snake_case						:	Tuple							,  __snake_case						:	Union[str, Any]							):





											'''simple docstring'''





											_snake_case: Union[str, Any]	       =		ImageClassificationPipeline(model=__snake_case							,  image_processor=__snake_case							,  top_k=2							)
											_snake_case: Tuple	       =		[
											    Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'							),
											    'http://images.cocodataset.org/val2017/000000039769.jpg',
											]
											return image_classifier, examples


					def  SCREAMING_SNAKE_CASE_		(       self						:	Dict							,  __snake_case						:	Any							,  __snake_case						:	str							):





											'''simple docstring'''





											_snake_case: Dict	       =		image_classifier('./tests/fixtures/tests_samples/COCO/000000039769.png'							)

											self.assertEqual(
											    __snake_case							,  [
											        {'score': ANY(__snake_case							), 'label': ANY(__snake_case							)},
											        {'score': ANY(__snake_case							), 'label': ANY(__snake_case							)},
											    ]							,  )

											import datasets

											_snake_case: str	       =		datasets.load_dataset('hf-internal-testing/fixtures_image_utils'							,  'image'							,  split='test'							)

											# Accepts URL + PIL.Image + lists
											_snake_case: List[str]	       =		image_classifier(
											    [
											        Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'							),
											        'http://images.cocodataset.org/val2017/000000039769.jpg',
											        # RGBA
											        dataset[0]['file'],
											        # LA
											        dataset[1]['file'],
											        # L
											        dataset[2]['file'],
											    ]							)
											self.assertEqual(
											    __snake_case							,  [
											        [
											            {'score': ANY(__snake_case							), 'label': ANY(__snake_case							)},
											            {'score': ANY(__snake_case							), 'label': ANY(__snake_case							)},
											        ],
											        [
											            {'score': ANY(__snake_case							), 'label': ANY(__snake_case							)},
											            {'score': ANY(__snake_case							), 'label': ANY(__snake_case							)},
											        ],
											        [
											            {'score': ANY(__snake_case							), 'label': ANY(__snake_case							)},
											            {'score': ANY(__snake_case							), 'label': ANY(__snake_case							)},
											        ],
											        [
											            {'score': ANY(__snake_case							), 'label': ANY(__snake_case							)},
											            {'score': ANY(__snake_case							), 'label': ANY(__snake_case							)},
											        ],
											        [
											            {'score': ANY(__snake_case							), 'label': ANY(__snake_case							)},
											            {'score': ANY(__snake_case							), 'label': ANY(__snake_case							)},
											        ],
											    ]							,  )


					@require_torch
					def  SCREAMING_SNAKE_CASE_		(       self						:	Optional[int]							):





											'''simple docstring'''





											_snake_case: Any	       =		'hf-internal-testing/tiny-random-vit'
											_snake_case: List[str]	       =		pipeline('image-classification'							,  model=__snake_case							)

											_snake_case: List[str]	       =		image_classifier('http://images.cocodataset.org/val2017/000000039769.jpg'							)
											self.assertEqual(
											    nested_simplify(__snake_case							,  decimals=4							)							,  [{'label': 'LABEL_1', 'score': 0.574}, {'label': 'LABEL_0', 'score': 0.426}]							,  )

											_snake_case: Dict	       =		image_classifier(
											    [
											        'http://images.cocodataset.org/val2017/000000039769.jpg',
											        'http://images.cocodataset.org/val2017/000000039769.jpg',
											    ]							,  top_k=2							,  )
											self.assertEqual(
											    nested_simplify(__snake_case							,  decimals=4							)							,  [
											        [{'label': 'LABEL_1', 'score': 0.574}, {'label': 'LABEL_0', 'score': 0.426}],
											        [{'label': 'LABEL_1', 'score': 0.574}, {'label': 'LABEL_0', 'score': 0.426}],
											    ]							,  )


					@require_tf
					def  SCREAMING_SNAKE_CASE_		(       self						:	List[str]							):





											'''simple docstring'''





											_snake_case: List[str]	       =		'hf-internal-testing/tiny-random-vit'
											_snake_case: List[str]	       =		pipeline('image-classification'							,  model=__snake_case							,  framework='tf'							)

											_snake_case: Any	       =		image_classifier('http://images.cocodataset.org/val2017/000000039769.jpg'							)
											self.assertEqual(
											    nested_simplify(__snake_case							,  decimals=4							)							,  [{'label': 'LABEL_1', 'score': 0.574}, {'label': 'LABEL_0', 'score': 0.426}]							,  )

											_snake_case: str	       =		image_classifier(
											    [
											        'http://images.cocodataset.org/val2017/000000039769.jpg',
											        'http://images.cocodataset.org/val2017/000000039769.jpg',
											    ]							,  top_k=2							,  )
											self.assertEqual(
											    nested_simplify(__snake_case							,  decimals=4							)							,  [
											        [{'label': 'LABEL_1', 'score': 0.574}, {'label': 'LABEL_0', 'score': 0.426}],
											        [{'label': 'LABEL_1', 'score': 0.574}, {'label': 'LABEL_0', 'score': 0.426}],
											    ]							,  )


					def  SCREAMING_SNAKE_CASE_		(       self						:	Tuple							):





											'''simple docstring'''





											_snake_case: Union[str, Any]	       =		PreTrainedTokenizer()

											# Assert that the pipeline can be initialized with a feature extractor that is not in any mapping
											_snake_case: Any	       =		pipeline(
											    'image-classification'							,  model='hf-internal-testing/tiny-random-vit'							,  tokenizer=__snake_case							)

											self.assertIs(image_classifier.tokenizer							,  __snake_case							)




					@slow
					@require_torch
					def  SCREAMING_SNAKE_CASE_		(       self						:	List[str]							):





											'''simple docstring'''





											_snake_case: Any	       =		pipeline('image-classification'							,  model='deepmind/vision-perceiver-conv'							)
											_snake_case: int	       =		image_classifier('http://images.cocodataset.org/val2017/000000039769.jpg'							)
											self.assertEqual(
											    nested_simplify(__snake_case							,  decimals=4							)							,  [
											        {'score': 0.4_385, 'label': 'tabby, tabby cat'},
											        {'score': 0.321, 'label': 'tiger cat'},
											        {'score': 0.0_502, 'label': 'Egyptian cat'},
											        {'score': 0.0_137, 'label': 'crib, cot'},
											        {'score': 0.007, 'label': 'radiator'},
											    ]							,  )

											_snake_case: Union[str, Any]	       =		pipeline('image-classification'							,  model='deepmind/vision-perceiver-fourier'							)
											_snake_case: int	       =		image_classifier('http://images.cocodataset.org/val2017/000000039769.jpg'							)
											self.assertEqual(
											    nested_simplify(__snake_case							,  decimals=4							)							,  [
											        {'score': 0.5_658, 'label': 'tabby, tabby cat'},
											        {'score': 0.1_309, 'label': 'tiger cat'},
											        {'score': 0.0_722, 'label': 'Egyptian cat'},
											        {'score': 0.0_707, 'label': 'remote control, remote'},
											        {'score': 0.0_082, 'label': 'computer keyboard, keypad'},
											    ]							,  )

											_snake_case: Optional[int]	       =		pipeline('image-classification'							,  model='deepmind/vision-perceiver-learned'							)
											_snake_case: str	       =		image_classifier('http://images.cocodataset.org/val2017/000000039769.jpg'							)
											self.assertEqual(
											    nested_simplify(__snake_case							,  decimals=4							)							,  [
											        {'score': 0.3_022, 'label': 'tabby, tabby cat'},
											        {'score': 0.2_362, 'label': 'Egyptian cat'},
											        {'score': 0.1_856, 'label': 'tiger cat'},
											        {'score': 0.0_324, 'label': 'remote control, remote'},
											        {'score': 0.0_096, 'label': 'quilt, comforter, comfort, puff'},
											    ]							,  )





