"""simple docstring"""






import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin





class  _A						(       lowerCAmelCase  ,    unittest.TestCase	):
      snake_case__			:   Optional[Any]							  =	CanineTokenizer
      snake_case__			:   str							  =	False

      def 				A__       (						self						):







            """simple docstring"""



            super().setUp()
            lowercase														=	CanineTokenizer()
            tokenizer.save_pretrained(self.tmpdirname						)

      @cached_property
      def 				A__       (						self						):







            """simple docstring"""



            return CanineTokenizer.from_pretrained("""google/canine-s"""						)

      def 				A__       (						self		,       **__lowerCAmelCase						):







            """simple docstring"""



            lowercase														=	self.tokenizer_class.from_pretrained(self.tmpdirname		,       **__lowerCAmelCase						)
            lowercase														=	1024
            return tokenizer

      @require_torch
      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	self.canine_tokenizer
            lowercase														=	["""Life is like a box of chocolates.""", """You never know what you're gonna get."""]
            # fmt: off
            lowercase														=	[5_7344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 5_7345, 0, 0, 0, 0]
            # fmt: on
            lowercase														=	tokenizer(__lowerCAmelCase		,       padding=__lowerCAmelCase		,       return_tensors="""pt"""						)
            self.assertIsInstance(__lowerCAmelCase		,       __lowerCAmelCase						)

            lowercase														=	list(batch.input_ids.numpy()[0]						)

            self.assertListEqual(__lowerCAmelCase		,       __lowerCAmelCase						)

            self.assertEqual((2, 39)		,       batch.input_ids.shape						)
            self.assertEqual((2, 39)		,       batch.attention_mask.shape						)

      @require_torch
      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	self.canine_tokenizer
            lowercase														=	["""Once there was a man.""", """He wrote a test in HuggingFace Tranformers."""]
            lowercase														=	tokenizer(__lowerCAmelCase		,       padding=__lowerCAmelCase		,       return_tensors="""pt"""						)
            # check if input_ids, attention_mask and token_type_ids are returned
            self.assertIn("""input_ids"""		,       __lowerCAmelCase						)
            self.assertIn("""attention_mask"""		,       __lowerCAmelCase						)
            self.assertIn("""token_type_ids"""		,       __lowerCAmelCase						)

      @require_torch
      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	self.canine_tokenizer
            lowercase														=	[
                """What's the weater?""",
                """It's about 25 degrees.""",
            ]
            lowercase														=	tokenizer(
                text_target=__lowerCAmelCase		,       max_length=32		,       padding="""max_length"""		,       truncation=__lowerCAmelCase		,       return_tensors="""pt"""						)
            self.assertEqual(32		,       targets["""input_ids"""].shape[1]						)

      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	self.get_tokenizers()
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'						):
                        self.assertNotEqual(tokenizer.model_max_length		,       42						)

        # Now let's start the test
            lowercase														=	self.get_tokenizers()
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'						):
                        # Isolate this from the other tests because we save additional tokens/etc
                        lowercase														=	tempfile.mkdtemp()

                        lowercase														=	""" He is very happy, UNwant\u00E9d,running"""
                        lowercase														=	tokenizer.encode(__lowerCAmelCase		,       add_special_tokens=__lowerCAmelCase						)
                        tokenizer.save_pretrained(__lowerCAmelCase						)

                        lowercase														=	tokenizer.__class__.from_pretrained(__lowerCAmelCase						)
                        lowercase														=	after_tokenizer.encode(__lowerCAmelCase		,       add_special_tokens=__lowerCAmelCase						)
                        self.assertListEqual(__lowerCAmelCase		,       __lowerCAmelCase						)

                        shutil.rmtree(__lowerCAmelCase						)

            lowercase														=	self.get_tokenizers(model_max_length=42						)
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'						):
                        # Isolate this from the other tests because we save additional tokens/etc
                        lowercase														=	tempfile.mkdtemp()

                        lowercase														=	""" He is very happy, UNwant\u00E9d,running"""

                        lowercase														=	tokenizer.additional_special_tokens

                        # We can add a new special token for Canine as follows:
                        lowercase														=	chr(0Xe_007						)
                        additional_special_tokens.append(__lowerCAmelCase						)
                        tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}						)
                        lowercase														=	tokenizer.encode(__lowerCAmelCase		,       add_special_tokens=__lowerCAmelCase						)
                        tokenizer.save_pretrained(__lowerCAmelCase						)

                        lowercase														=	tokenizer.__class__.from_pretrained(__lowerCAmelCase						)
                        lowercase														=	after_tokenizer.encode(__lowerCAmelCase		,       add_special_tokens=__lowerCAmelCase						)
                        self.assertListEqual(__lowerCAmelCase		,       __lowerCAmelCase						)
                        self.assertIn(__lowerCAmelCase		,       after_tokenizer.additional_special_tokens						)
                        self.assertEqual(after_tokenizer.model_max_length		,       42						)

                        lowercase														=	tokenizer.__class__.from_pretrained(__lowerCAmelCase		,       model_max_length=43						)
                        self.assertEqual(tokenizer.model_max_length		,       43						)

                        shutil.rmtree(__lowerCAmelCase						)

      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	self.get_tokenizers(do_lower_case=__lowerCAmelCase						)
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'						):
                        lowercase							,							lowercase														=	self.get_clean_sequence(__lowerCAmelCase						)

                        # a special token for Canine can be defined as follows:
                        lowercase														=	0Xe_005
                        lowercase														=	chr(__lowerCAmelCase						)

                        tokenizer.add_special_tokens({"""cls_token""": special_token}						)
                        lowercase														=	tokenizer.encode(__lowerCAmelCase		,       add_special_tokens=__lowerCAmelCase						)
                        self.assertEqual(len(__lowerCAmelCase						)		,       1						)

                        lowercase														=	tokenizer.decode(ids + encoded_special_token		,       clean_up_tokenization_spaces=__lowerCAmelCase						)
                        lowercase														=	tokenizer.encode(__lowerCAmelCase		,       add_special_tokens=__lowerCAmelCase						)

                        lowercase														=	tokenizer.encode(__lowerCAmelCase		,       add_special_tokens=__lowerCAmelCase						)
                        lowercase														=	tokenizer.encode(__lowerCAmelCase		,       add_special_tokens=__lowerCAmelCase						)
                        self.assertEqual(__lowerCAmelCase		,       input_encoded + special_token_id						)

                        lowercase														=	tokenizer.decode(__lowerCAmelCase		,       skip_special_tokens=__lowerCAmelCase						)
                        self.assertTrue(special_token not in decoded						)

      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	self.get_tokenizers(do_lower_case=__lowerCAmelCase						)
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'						):
                        lowercase														=	chr(0Xe_005						)
                        lowercase														=	chr(0Xe_006						)

                        # `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
                        tokenizer.add_tokens([SPECIAL_TOKEN_1]		,       special_tokens=__lowerCAmelCase						)
                        # `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
                        # which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
                        tokenizer.add_special_tokens({"""additional_special_tokens""": [SPECIAL_TOKEN_2]}						)

                        lowercase														=	tokenizer.tokenize(__lowerCAmelCase						)
                        lowercase														=	tokenizer.tokenize(__lowerCAmelCase						)

                        self.assertEqual(len(__lowerCAmelCase						)		,       1						)
                        self.assertEqual(len(__lowerCAmelCase						)		,       1						)
                        self.assertEqual(token_a[0]		,       __lowerCAmelCase						)
                        self.assertEqual(token_a[0]		,       __lowerCAmelCase						)

      @require_tokenizers
      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	self.get_tokenizers(do_lower_case=__lowerCAmelCase						)
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'						):
                        # a special token for Canine can be defined as follows:
                        lowercase														=	0Xe_006
                        lowercase														=	chr(__lowerCAmelCase						)

                        lowercase														=	AddedToken(__lowerCAmelCase		,       lstrip=__lowerCAmelCase						)
                        tokenizer.add_special_tokens({"""additional_special_tokens""": [new_token]}						)

                        with tempfile.TemporaryDirectory() as tmp_dir_name:
                              tokenizer.save_pretrained(__lowerCAmelCase						)
                              tokenizer.from_pretrained(__lowerCAmelCase						)

      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	[]
            if self.test_slow_tokenizer:
                  tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

            if self.test_rust_tokenizer:
                  tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

            for tokenizer_class, tokenizer_utils in tokenizer_list:
                  with tempfile.TemporaryDirectory() as tmp_dir:
                        tokenizer_utils.save_pretrained(__lowerCAmelCase						)

                        with open(os.path.join(__lowerCAmelCase		,       """special_tokens_map.json"""						)		,       encoding="""utf-8"""						) as json_file:
                              lowercase														=	json.load(__lowerCAmelCase						)

                        with open(os.path.join(__lowerCAmelCase		,       """tokenizer_config.json"""						)		,       encoding="""utf-8"""						) as json_file:
                              lowercase														=	json.load(__lowerCAmelCase						)

                        # a special token for Canine can be defined as follows:
                        lowercase														=	0Xe_006
                        lowercase														=	chr(__lowerCAmelCase						)

                        lowercase														=	[new_token_a]
                        lowercase														=	[new_token_a]

                        with open(os.path.join(__lowerCAmelCase		,       """special_tokens_map.json"""						)		,       """w"""		,       encoding="""utf-8"""						) as outfile:
                              json.dump(__lowerCAmelCase		,       __lowerCAmelCase						)
                        with open(os.path.join(__lowerCAmelCase		,       """tokenizer_config.json"""						)		,       """w"""		,       encoding="""utf-8"""						) as outfile:
                              json.dump(__lowerCAmelCase		,       __lowerCAmelCase						)

                        # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                        # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                        # "special_tokens_map.json" files
                        lowercase														=	tokenizer_class.from_pretrained(__lowerCAmelCase		,       extra_ids=0						)
                        self.assertIn(__lowerCAmelCase		,       tokenizer_without_change_in_init.additional_special_tokens						)
                        # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                        self.assertEqual(
                            [new_token_a]		,       tokenizer_without_change_in_init.convert_ids_to_tokens(
                                tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]						)						)		,       )

                        lowercase														=	0Xe_007
                        lowercase														=	chr(__lowerCAmelCase						)
                        # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                        lowercase														=	[AddedToken(__lowerCAmelCase		,       lstrip=__lowerCAmelCase						)]
                        lowercase														=	tokenizer_class.from_pretrained(
                            __lowerCAmelCase		,       additional_special_tokens=__lowerCAmelCase		,       extra_ids=0						)

                        self.assertIn(__lowerCAmelCase		,       tokenizer.additional_special_tokens						)
                        # self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
                        self.assertEqual(
                            [new_token_a]		,       tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]						)						)						)

      @require_tokenizers
      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	self.get_tokenizers(do_lower_case=__lowerCAmelCase						)
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'						):
                        lowercase														=	"""hello world"""
                        if self.space_between_special_tokens:
                              lowercase														=	"""[CLS] hello world [SEP]"""
                        else:
                              lowercase														=	input
                        lowercase														=	tokenizer.encode(__lowerCAmelCase		,       add_special_tokens=__lowerCAmelCase						)
                        lowercase														=	tokenizer.decode(__lowerCAmelCase		,       spaces_between_special_tokens=self.space_between_special_tokens						)
                        self.assertIn(__lowerCAmelCase		,       [output, output.lower()]						)

      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	self.get_tokenizers()
            for tokenizer in tokenizers:
                  with self.subTest(f'{tokenizer.__class__.__name__}'						):
                        lowercase														=	[
                            """bos_token""",
                            """eos_token""",
                            """unk_token""",
                            """sep_token""",
                            """pad_token""",
                            """cls_token""",
                            """mask_token""",
                        ]

            lowercase														=	"""a"""
            lowercase														=	ord(__lowerCAmelCase						)

            for attr in attributes_list:
                  setattr(__lowerCAmelCase		,       attr + """_id"""		,       __lowerCAmelCase						)
                  self.assertEqual(getattr(__lowerCAmelCase		,       __lowerCAmelCase						)		,       __lowerCAmelCase						)
                  self.assertEqual(getattr(__lowerCAmelCase		,       attr + """_id"""						)		,       __lowerCAmelCase						)

                  setattr(__lowerCAmelCase		,       attr + """_id"""		,       __lowerCAmelCase						)
                  self.assertEqual(getattr(__lowerCAmelCase		,       __lowerCAmelCase						)		,       __lowerCAmelCase						)
                  self.assertEqual(getattr(__lowerCAmelCase		,       attr + """_id"""						)		,       __lowerCAmelCase						)

            setattr(__lowerCAmelCase		,       """additional_special_tokens_ids"""		,       []						)
            self.assertListEqual(getattr(__lowerCAmelCase		,       """additional_special_tokens"""						)		,       []						)
            self.assertListEqual(getattr(__lowerCAmelCase		,       """additional_special_tokens_ids"""						)		,       []						)

            lowercase														=	0Xe_006
            lowercase														=	chr(__lowerCAmelCase						)
            setattr(__lowerCAmelCase		,       """additional_special_tokens_ids"""		,       [additional_special_token_id]						)
            self.assertListEqual(getattr(__lowerCAmelCase		,       """additional_special_tokens"""						)		,       [additional_special_token]						)
            self.assertListEqual(getattr(__lowerCAmelCase		,       """additional_special_tokens_ids"""						)		,       [additional_special_token_id]						)

      def 				A__       (						self						):







            """simple docstring"""



            pass

      def 				A__       (						self						):







            """simple docstring"""



            pass

      def 				A__       (						self						):







            """simple docstring"""



            pass

      def 				A__       (						self						):







            """simple docstring"""



            pass

      def 				A__       (						self						):







            """simple docstring"""



            pass

      def 				A__       (						self						):







            """simple docstring"""



            pass

      def 				A__       (						self						):







            """simple docstring"""



            pass







      def 				A__       (						self						):







            """simple docstring"""



            pass




"""simple docstring"""






from collections import OrderedDict
from typing import List, Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__lowerCAmelCase    :       List[Any]					     =logging.get_logger(__name__)

__lowerCAmelCase    :       Union[str, Any]					     ={
    """google/efficientnet-b7""": """https://huggingface.co/google/efficientnet-b7/resolve/main/config.json""",
}





class  _A						(       lowerCAmelCase	):
      snake_case__			:   Optional[int]							  =	'efficientnet'

      def __init__(						self		,       __lowerCAmelCase = 3		,       __lowerCAmelCase = 600		,       __lowerCAmelCase = 2.0		,       __lowerCAmelCase = 3.1		,       __lowerCAmelCase = 8		,       __lowerCAmelCase = [3, 3, 5, 3, 5, 5, 3]		,       __lowerCAmelCase = [32, 16, 24, 40, 80, 112, 192]		,       __lowerCAmelCase = [16, 24, 40, 80, 112, 192, 320]		,       __lowerCAmelCase = []		,       __lowerCAmelCase = [1, 2, 2, 2, 1, 2, 1]		,       __lowerCAmelCase = [1, 2, 2, 3, 3, 4, 1]		,       __lowerCAmelCase = [1, 6, 6, 6, 6, 6, 6]		,       __lowerCAmelCase = 0.2_5		,       __lowerCAmelCase = "swish"		,       __lowerCAmelCase = 2560		,       __lowerCAmelCase = "mean"		,       __lowerCAmelCase = 0.0_2		,       __lowerCAmelCase = 0.0_0_1		,       __lowerCAmelCase = 0.9_9		,       __lowerCAmelCase = 0.5		,       __lowerCAmelCase = 0.2		,       **__lowerCAmelCase		,       ):







            """simple docstring"""



            super().__init__(**__lowerCAmelCase						)

            lowercase														=	num_channels
            lowercase														=	image_size
            lowercase														=	width_coefficient
            lowercase														=	depth_coefficient
            lowercase														=	depth_divisor
            lowercase														=	kernel_sizes
            lowercase														=	in_channels
            lowercase														=	out_channels
            lowercase														=	depthwise_padding
            lowercase														=	strides
            lowercase														=	num_block_repeats
            lowercase														=	expand_ratios
            lowercase														=	squeeze_expansion_ratio
            lowercase														=	hidden_act
            lowercase														=	hidden_dim
            lowercase														=	pooling_type
            lowercase														=	initializer_range
            lowercase														=	batch_norm_eps
            lowercase														=	batch_norm_momentum
            lowercase														=	dropout_rate
            lowercase														=	drop_connect_rate
            lowercase														=	sum(__lowerCAmelCase						) * 4






class  _A						(       lowerCAmelCase	):
      snake_case__			:   Dict							  =	version.parse('1.11'	)

      @property
      def 				A__       (						self						):







            """simple docstring"""



            return OrderedDict(
                [
                    ("""pixel_values""", {0: """batch""", 1: """num_channels""", 2: """height""", 3: """width"""}),
                ]						)







      @property
      def 				A__       (						self						):







            """simple docstring"""



            return 1E-5



