




"""simple docstring"""




import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
			a_	   =    'pt'
elif is_tf_available():
			a_	   =    'tf'
else:
			a_	   =    'jax'

class    UpperCAmelCase_  (       snake_case					, unittest.TestCase						):
		UpperCamelCase						 =ByTaTokenizer
		UpperCamelCase						 =False
		def      _lowerCamelCase				(		self    )		->    Any:
					super().setUp()
					__lowercase       :   List[Any]        =    ByTaTokenizer()
					tokenizer.save_pretrained(self.tmpdirname    )
		@cached_property
		def      _lowerCamelCase				(		self    )		->    List[str]:
					return ByTaTokenizer.from_pretrained('''google/byt5-small'''    )
		def      _lowerCamelCase				(		self							,		**UpperCamelCase_    )		->    List[Any]:
					return self.tokenizer_class.from_pretrained(self.tmpdirname							,		**UpperCamelCase_    )
		def      _lowerCamelCase				(		self							,		UpperCamelCase_							,		UpperCamelCase_=False							,		UpperCamelCase_=20							,		UpperCamelCase_=5    )		->    Tuple:
					__lowercase       :   Optional[int]        =    []
					for i in range(len(UpperCamelCase_    )    ):
								try:
											__lowercase       :   Optional[Any]        =    tokenizer.decode([i]							,		clean_up_tokenization_spaces=UpperCamelCase_    )
								except UnicodeDecodeError:
											pass
								toks.append((i, tok)    )

					__lowercase       :   List[Any]        =    list(filter(lambda UpperCamelCase_    : re.match(R'''^[ a-zA-Z]+$'''							,		t[1]    )							,		UpperCamelCase_    )    )
					__lowercase       :   Dict        =    list(filter(lambda UpperCamelCase_    : [t[0]] == tokenizer.encode(t[1]							,		add_special_tokens=UpperCamelCase_    )							,		UpperCamelCase_    )    )
					if max_length is not None and len(UpperCamelCase_    ) > max_length:
								__lowercase       :   int        =    toks[:max_length]
					if min_length is not None and len(UpperCamelCase_    ) < min_length and len(UpperCamelCase_    ) > 0:
								while len(UpperCamelCase_    ) < min_length:
											__lowercase       :   str        =    toks + toks
        # toks_str = [t[1] for t in toks]
					__lowercase       :   Union[str, Any]        =    [t[0] for t in toks]

					# Ensure consistency
					__lowercase       :   Tuple        =    tokenizer.decode(UpperCamelCase_							,		clean_up_tokenization_spaces=UpperCamelCase_    )
					if " " not in output_txt and len(UpperCamelCase_    ) > 1:
								__lowercase       :   int        =    (
								    tokenizer.decode([toks_ids[0]]							,		clean_up_tokenization_spaces=UpperCamelCase_    )
								    + ' '
								    + tokenizer.decode(toks_ids[1:]							,		clean_up_tokenization_spaces=UpperCamelCase_    )
								)
					if with_prefix_space:
								__lowercase       :   List[str]        =    ' ' + output_txt
					__lowercase       :   Optional[Any]        =    tokenizer.encode(UpperCamelCase_							,		add_special_tokens=UpperCamelCase_    )
					return output_txt, output_ids
		def      _lowerCamelCase				(		self    )		->    Optional[Any]:
					__lowercase       :   List[Any]        =    self.ta_base_tokenizer
					__lowercase       :   List[str]        =    tokenizer(['''hi</s>''', '''I went to the gym</s>''', '''</s>''']    )
					__lowercase       :   str        =    tokenizer(['''hi''', '''I went to the gym''', '''''']    )
					self.assertListEqual(batch_with_eos_added['''input_ids''']							,		batch_without_eos_added['''input_ids''']    )
		def      _lowerCamelCase				(		self    )		->    Tuple:
					__lowercase       :   List[Any]        =    self.ta_base_tokenizer
					__lowercase       :   Tuple        =    'Unicode €.'
					__lowercase       :   int        =    tokenizer(UpperCamelCase_    )
					__lowercase       :   Tuple        =    [88, 1_13, 1_08, 1_02, 1_14, 1_03, 1_04, 35, 2_29, 1_33, 1_75, 49, 1]
					self.assertEqual(encoded['''input_ids''']							,		UpperCamelCase_    )

					# decoding
					__lowercase       :   int        =    tokenizer.decode(UpperCamelCase_    )
					self.assertEqual(UpperCamelCase_							,		'''Unicode €.</s>'''    )

					__lowercase       :   int        =    tokenizer('''e è é ê ë'''    )
					__lowercase       :   Dict        =    [1_04, 35, 1_98, 1_71, 35, 1_98, 1_72, 35, 1_98, 1_73, 35, 1_98, 1_74, 1]
					self.assertEqual(encoded['''input_ids''']							,		UpperCamelCase_    )
					# decoding
					__lowercase       :   int        =    tokenizer.decode(UpperCamelCase_    )
					self.assertEqual(UpperCamelCase_							,		'''e è é ê ë</s>'''    )

					# encode/decode, but with `encode` instead of `__call__`
					self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''    )    )							,		'''e è é ê ë</s>'''    )
		def      _lowerCamelCase				(		self    )		->    List[str]:
					__lowercase       :   int        =    self.ta_base_tokenizer
					__lowercase       :   Optional[Any]        =    ['A long paragraph for summarization.', 'Another paragraph for summarization.']
					# fmt: off
					__lowercase       :   str        =    [68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 1, 0]
					# fmt: on
					__lowercase       :   Dict        =    tokenizer(UpperCamelCase_							,		padding=UpperCamelCase_							,		return_tensors=UpperCamelCase_    )
					self.assertIsInstance(UpperCamelCase_							,		UpperCamelCase_    )

					if FRAMEWORK != "jax":
								__lowercase       :   Dict        =    list(batch.input_ids.numpy()[0]    )
					else:
								__lowercase       :   int        =    list(batch.input_ids.tolist()[0]    )

					self.assertListEqual(UpperCamelCase_							,		UpperCamelCase_    )

					self.assertEqual((2, 37)							,		batch.input_ids.shape    )
					self.assertEqual((2, 37)							,		batch.attention_mask.shape    )
		def      _lowerCamelCase				(		self    )		->    str:
					__lowercase       :   Union[str, Any]        =    self.ta_base_tokenizer
					__lowercase       :   List[Any]        =    ['A long paragraph for summarization.', 'Another paragraph for summarization.']
					__lowercase       :   Any        =    tokenizer(UpperCamelCase_							,		padding=UpperCamelCase_							,		return_tensors=UpperCamelCase_    )
					# check if input_ids are returned and no decoder_input_ids
					self.assertIn('''input_ids'''							,		UpperCamelCase_    )
					self.assertIn('''attention_mask'''							,		UpperCamelCase_    )
					self.assertNotIn('''decoder_input_ids'''							,		UpperCamelCase_    )
					self.assertNotIn('''decoder_attention_mask'''							,		UpperCamelCase_    )
		def      _lowerCamelCase				(		self    )		->    Tuple:
					__lowercase       :   List[str]        =    self.ta_base_tokenizer
					__lowercase       :   Tuple        =    [
					    'Summary of the text.',
					    'Another summary.',
					]
					__lowercase       :   Any        =    tokenizer(
					    text_target=UpperCamelCase_							,		max_length=32							,		padding='''max_length'''							,		truncation=UpperCamelCase_							,		return_tensors=UpperCamelCase_    )
					self.assertEqual(32							,		targets['''input_ids'''].shape[1]    )
		def      _lowerCamelCase				(		self    )		->    Optional[Any]:
					__lowercase       :   Optional[Any]        =    self.ta_base_tokenizer
					__lowercase       :   str        =    ['A long paragraph for summarization. </s>']
					__lowercase       :   Optional[Any]        =    ['Summary of the text. </s>']
					# fmt: off
					__lowercase       :   Optional[int]        =    [68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 35, 1]
					__lowercase       :   Union[str, Any]        =    [86, 1_20, 1_12, 1_12, 1_00, 1_17, 1_24, 35, 1_14, 1_05, 35, 1_19, 1_07, 1_04, 35, 1_19, 1_04, 1_23, 1_19, 49, 35, 1]
					# fmt: on

					__lowercase       :   str        =    tokenizer(UpperCamelCase_							,		text_target=UpperCamelCase_    )

					self.assertEqual(UpperCamelCase_							,		batch['''input_ids'''][0]    )
					self.assertEqual(UpperCamelCase_							,		batch['''labels'''][0]    )
		def      _lowerCamelCase				(		self    )		->    Dict:
					__lowercase       :   Union[str, Any]        =    self.get_tokenizers()
					for tokenizer in tokenizers:
								with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
											self.assertNotEqual(tokenizer.model_max_length							,		42    )

        # Now let's start the test
					__lowercase       :   List[str]        =    self.get_tokenizers()
					for tokenizer in tokenizers:
								with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
											# Isolate this from the other tests because we save additional tokens/etc
											__lowercase       :   Any        =    tempfile.mkdtemp()

											__lowercase       :   str        =    ' He is very happy, UNwant\u00E9d,running'
											__lowercase       :   Any        =    tokenizer.encode(UpperCamelCase_							,		add_special_tokens=UpperCamelCase_    )
											tokenizer.save_pretrained(UpperCamelCase_    )

											__lowercase       :   Dict        =    tokenizer.__class__.from_pretrained(UpperCamelCase_    )
											__lowercase       :   List[Any]        =    after_tokenizer.encode(UpperCamelCase_							,		add_special_tokens=UpperCamelCase_    )
											self.assertListEqual(UpperCamelCase_							,		UpperCamelCase_    )

											shutil.rmtree(UpperCamelCase_    )

					__lowercase       :   Optional[int]        =    self.get_tokenizers(model_max_length=42    )
					for tokenizer in tokenizers:
								with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
											# Isolate this from the other tests because we save additional tokens/etc
											__lowercase       :   int        =    tempfile.mkdtemp()

											__lowercase       :   Union[str, Any]        =    ' He is very happy, UNwant\u00E9d,running'
											tokenizer.add_tokens(['''bim''', '''bambam''']    )
											__lowercase       :   Optional[Any]        =    tokenizer.additional_special_tokens
											additional_special_tokens.append('''new_additional_special_token'''    )
											tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}    )
											__lowercase       :   Dict        =    tokenizer.encode(UpperCamelCase_							,		add_special_tokens=UpperCamelCase_    )
											tokenizer.save_pretrained(UpperCamelCase_    )

											__lowercase       :   List[Any]        =    tokenizer.__class__.from_pretrained(UpperCamelCase_    )
											__lowercase       :   Dict        =    after_tokenizer.encode(UpperCamelCase_							,		add_special_tokens=UpperCamelCase_    )
											self.assertListEqual(UpperCamelCase_							,		UpperCamelCase_    )
											self.assertIn('''new_additional_special_token'''							,		after_tokenizer.additional_special_tokens    )
											self.assertEqual(after_tokenizer.model_max_length							,		42    )

											__lowercase       :   str        =    tokenizer.__class__.from_pretrained(UpperCamelCase_							,		model_max_length=43    )
											self.assertEqual(tokenizer.model_max_length							,		43    )

											shutil.rmtree(UpperCamelCase_    )
		def      _lowerCamelCase				(		self    )		->    Optional[Any]:
					__lowercase       :   Tuple        =    []
					if self.test_slow_tokenizer:
								tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())    )

					if self.test_rust_tokenizer:
								tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())    )

					for tokenizer_class, tokenizer_utils in tokenizer_list:
								with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer_utils.save_pretrained(UpperCamelCase_    )

											with open(os.path.join(UpperCamelCase_							,		'''special_tokens_map.json'''    )							,		encoding='''utf-8'''    ) as json_file:
														__lowercase       :   int        =    json.load(UpperCamelCase_    )

											with open(os.path.join(UpperCamelCase_							,		'''tokenizer_config.json'''    )							,		encoding='''utf-8'''    ) as json_file:
														__lowercase       :   Optional[Any]        =    json.load(UpperCamelCase_    )

											__lowercase       :   List[Any]        =    [F"""<extra_id_{i}>""" for i in range(1_25    )]

											__lowercase       :   Dict        =    added_tokens_extra_ids + [
											    'an_additional_special_token'
											]
											__lowercase       :   Optional[int]        =    added_tokens_extra_ids + [
											    'an_additional_special_token'
											]

											with open(os.path.join(UpperCamelCase_							,		'''special_tokens_map.json'''    )							,		'''w'''							,		encoding='''utf-8'''    ) as outfile:
														json.dump(UpperCamelCase_							,		UpperCamelCase_    )
											with open(os.path.join(UpperCamelCase_							,		'''tokenizer_config.json'''    )							,		'''w'''							,		encoding='''utf-8'''    ) as outfile:
														json.dump(UpperCamelCase_							,		UpperCamelCase_    )

											# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
											# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
											# "special_tokens_map.json" files
											__lowercase       :   int        =    tokenizer_class.from_pretrained(
											    UpperCamelCase_							,		)
											self.assertIn(
											    '''an_additional_special_token'''							,		tokenizer_without_change_in_init.additional_special_tokens    )
											# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
											self.assertEqual(
											    ['''an_additional_special_token''']							,		tokenizer_without_change_in_init.convert_ids_to_tokens(
											        tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']    )    )							,		)

											# Now we test that we can change the value of additional_special_tokens in the from_pretrained
											__lowercase       :   str        =    added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''							,		lstrip=UpperCamelCase_    )]
											__lowercase       :   Tuple        =    tokenizer_class.from_pretrained(
											    UpperCamelCase_							,		additional_special_tokens=UpperCamelCase_							,		)

											self.assertIn('''a_new_additional_special_token'''							,		tokenizer.additional_special_tokens    )
											self.assertEqual(
											    ['''a_new_additional_special_token''']							,		tokenizer.convert_ids_to_tokens(
											        tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']    )    )							,		)
		def      _lowerCamelCase				(		self    )		->    List[Any]:
					__lowercase       :   int        =    []
					if self.test_slow_tokenizer:
								tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())    )

					if self.test_rust_tokenizer:
								tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())    )

					for tokenizer_class, tokenizer_utils in tokenizer_list:
								with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer_utils.save_pretrained(UpperCamelCase_    )

											__lowercase       :   List[str]        =    tokenizer_class.from_pretrained(UpperCamelCase_    )

											self.assertTrue(tokenizer.decode([2_55]    ) == ''''''    )
		def      _lowerCamelCase				(		self    )		->    Tuple:
					pass
		def      _lowerCamelCase				(		self    )		->    int:
					pass
		def      _lowerCamelCase				(		self    )		->    str:
					pass
		def      _lowerCamelCase				(		self    )		->    Optional[int]:
					pass
		def      _lowerCamelCase				(		self    )		->    List[Any]:
					__lowercase       :   int        =    self.get_tokenizers(fast=UpperCamelCase_							,		do_lower_case=UpperCamelCase_    )
					for tokenizer in tokenizers:
								with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
											__lowercase       :   List[str]        =    ['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
											__lowercase       :   Optional[Any]        =    tokenizer.convert_tokens_to_string(UpperCamelCase_    )

											self.assertIsInstance(UpperCamelCase_							,		UpperCamelCase_    )
		def      _lowerCamelCase				(		self    )		->    Any:
					__lowercase       :   str        =    self.get_tokenizers()
					for tokenizer in tokenizers:
								with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
											__lowercase       :   List[Any]        =    [
											    'bos_token',
											    'eos_token',
											    'unk_token',
											    'sep_token',
											    'pad_token',
											    'cls_token',
											    'mask_token',
											]

											__lowercase       :   List[Any]        =    0
											__lowercase       :   Dict        =    tokenizer.convert_ids_to_tokens(
											    UpperCamelCase_							,		skip_special_tokens=UpperCamelCase_    )

											for attr in attributes_list:
														setattr(UpperCamelCase_							,		attr + '''_id'''							,		UpperCamelCase_    )
														self.assertEqual(getattr(UpperCamelCase_							,		UpperCamelCase_    )							,		UpperCamelCase_    )
														self.assertEqual(getattr(UpperCamelCase_							,		attr + '''_id'''    )							,		UpperCamelCase_    )

														setattr(UpperCamelCase_							,		attr + '''_id'''							,		UpperCamelCase_    )
														self.assertEqual(getattr(UpperCamelCase_							,		UpperCamelCase_    )							,		UpperCamelCase_    )
														self.assertEqual(getattr(UpperCamelCase_							,		attr + '''_id'''    )							,		UpperCamelCase_    )

											setattr(UpperCamelCase_							,		'''additional_special_tokens_ids'''							,		[]    )
											self.assertListEqual(getattr(UpperCamelCase_							,		'''additional_special_tokens'''    )							,		[]    )
											self.assertListEqual(getattr(UpperCamelCase_							,		'''additional_special_tokens_ids'''    )							,		[]    )

											setattr(UpperCamelCase_							,		'''additional_special_tokens_ids'''							,		[token_id_to_test_setters]    )
											self.assertListEqual(getattr(UpperCamelCase_							,		'''additional_special_tokens'''    )							,		[token_to_test_setters]    )
											self.assertListEqual(getattr(UpperCamelCase_							,		'''additional_special_tokens_ids'''    )							,		[token_id_to_test_setters]    )





from typing import List, Union

from ..utils import (
    add_end_docstrings,
    is_tf_available,
    is_torch_available,
    is_vision_available,
    logging,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_vision_available():
					from PIL import Image

					from ..image_utils import load_image

if is_tf_available():
					import tensorflow as tf

					from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
					from ..tf_utils import stable_softmax

if is_torch_available():
					from ..models.auto.modeling_auto import MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING

lowerCamelCase__          =		logging.get_logger(__name__)







@add_end_docstrings(__magic_name__	)
class      A__							(     __magic_name__	):






				def __init__(						self					:      int    ,     *a					:      Dict    ,     **a					:      Union[str, Any]   ):






							'''simple docstring'''







							super().__init__(*a    ,     **a   )
							requires_backends(self    ,     'vision'   )
							self.check_model_type(
							    TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
							    if self.framework == 'tf'
							    else MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING   )






				def 							_lowerCamelCase		(						self					:      Dict    ,     a					:      List[str]=None   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Any         =		{}
							if top_k is not None:
										lowerCAmelCase__      :      Tuple         =		top_k
							return {}, {}, postprocess_params






				def __call__(						self					:      Any    ,     a					:      Union[str, List[str], "Image.Image", List["Image.Image"]]    ,     **a					:      List[Any]   ):






							'''simple docstring'''







							return super().__call__(a    ,     **a   )






				def 							_lowerCamelCase		(						self					:      Any    ,     a					:      Optional[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[Any]         =		load_image(a   )
							lowerCAmelCase__      :      Optional[int]         =		self.image_processor(images=a    ,     return_tensors=self.framework   )
							return model_inputs






				def 							_lowerCamelCase		(						self					:      Optional[int]    ,     a					:      List[str]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[Any]         =		self.model(**a   )
							return model_outputs






				def 							_lowerCamelCase		(						self					:      Optional[Any]    ,     a					:      List[Any]    ,     a					:      List[Any]=5   ):






							'''simple docstring'''







							if top_k > self.model.config.num_labels:
										lowerCAmelCase__      :      Optional[int]         =		self.model.config.num_labels

							if self.framework == "pt":
										lowerCAmelCase__      :      List[Any]         =		model_outputs.logits.softmax(-1   )[0]
										lowerCAmelCase__      ,				lowerCAmelCase__      :      Optional[int]         =		probs.topk(a   )
							elif self.framework == "tf":
										lowerCAmelCase__      :      Any         =		stable_softmax(model_outputs.logits    ,     axis=-1   )[0]
										lowerCAmelCase__      :      Any         =		tf.math.top_k(a    ,     k=a   )
										lowerCAmelCase__      ,				lowerCAmelCase__      :      int         =		topk.values.numpy(), topk.indices.numpy()
							else:
										raise ValueError(f'''Unsupported framework: {self.framework}'''   )

							lowerCAmelCase__      :      List[Any]         =		scores.tolist()
							lowerCAmelCase__      :      List[str]         =		ids.tolist()
							return [{"score": score, "label": self.model.config.idalabel[_id]} for score, _id in zip(a    ,     a   )]