



import argparse
import json
from dataclasses import dataclass, field
from functools import partial
from pathlib import Path
from typing import Callable, Dict, List, Tuple

import timm
import torch
import torch.nn as nn
from classy_vision.models.regnet import RegNet, RegNetParams, RegNetYaagf, RegNetYaagf, RegNetYaaagf
from huggingface_hub import cached_download, hf_hub_url
from torch import Tensor
from vissl.models.model_helpers import get_trunk_forward_outputs

from transformers import AutoImageProcessor, RegNetConfig, RegNetForImageClassification, RegNetModel
from transformers.utils import logging


logging.set_verbosity_info()
UpperCamelCase_										=					logging.get_logger()







@dataclass
class 	_snake_case							:






   '''simple docstring'''






   A__	:      Dict    =		42
   A__	:      Any    =		field(default_factory=__a      )
   A__	:      Dict    =		field(default_factory=__a      )


   def 	A__  (  self:			List[str]   ,lowerCamelCase_:			Dict   ,lowerCamelCase_:			List[str]   ,lowerCamelCase_:			Union[str, Any]				)     ->	Optional[Any]:
       UpperCAmelCase_			:    Union[str, Any]   						= len(list(m.modules()				)				) == 1 or isinstance(_UpperCamelCase   ,nn.Convad				) or isinstance(_UpperCamelCase   ,nn.BatchNormad				)
       if has_not_submodules:
           self.traced.append(_UpperCamelCase				)


   def __call__(  self:			Union[str, Any]   ,lowerCamelCase_:			Any				)     ->	str:
       for m in self.module.modules():
           self.handles.append(m.register_forward_hook(self._forward_hook				)				)
       self.module(_UpperCamelCase				)
       [x.remove() for x in self.handles]
       return self






   @property
   def 	A__  (  self:			Any				)     ->	str:
       # check the len of the state_dict keys to see if we have learnable params
       return list(filter(lambda lowerCamelCase_				: len(list(x.state_dict().keys()				)				) > 0   ,self.traced				)				)







@dataclass
class 	_snake_case							:






   '''simple docstring'''






   A__	:      Any    =		42
   A__	:      Any    =		42
   A__	:      Dict    =		1
   A__	:      Tuple    =		field(default_factory=__a      )
   A__	:      int    =		field(default_factory=__a      )
   A__	:      str    =		True


   def __call__(  self:			Union[str, Any]   ,lowerCamelCase_:			Tuple				)     ->	List[Any]:
       UpperCAmelCase_			:    Tuple   						= Tracker(self.dest				)(_UpperCamelCase				).parametrized
       UpperCAmelCase_			:    int   						= Tracker(self.src				)(_UpperCamelCase				).parametrized

       UpperCAmelCase_			:    List[str]   						= list(filter(lambda lowerCamelCase_				: type(_UpperCamelCase				) not in self.src_skip   ,_UpperCamelCase				)				)
       UpperCAmelCase_			:    List[str]   						= list(filter(lambda lowerCamelCase_				: type(_UpperCamelCase				) not in self.dest_skip   ,_UpperCamelCase				)				)

       if len(_UpperCamelCase				) != len(_UpperCamelCase				) and self.raise_if_mismatch:
           raise Exception(
               F'''Numbers of operations are different. Source module has {len(_UpperCamelCase				)} operations while'''
               F''' destination module has {len(_UpperCamelCase				)}.'''				)

       for dest_m, src_m in zip(_UpperCamelCase   ,_UpperCamelCase				):
           dest_m.load_state_dict(src_m.state_dict()				)
           if self.verbose == 1:
               print(F'''Transfered from={src_m} to={dest_m}'''				)







class 	_snake_case							(			nn.Module      ):






   '''simple docstring'''






   def __init__(  self:			int   ,lowerCamelCase_:			Optional[int]				)     ->	Dict:
       super().__init__()

       UpperCAmelCase_			:    Dict   						= []
       # - get the stem
       feature_blocks.append(("""conv1""", model.stem)				)
       # - get all the feature blocks
       for k, v in model.trunk_output.named_children():
           assert k.startswith("""block"""				), F'''Unexpected layer name {k}'''
           UpperCAmelCase_			:    int   						= len(_UpperCamelCase				) + 1
           feature_blocks.append((F'''res{block_index}''', v)				)

       UpperCAmelCase_			:    Any   						= nn.ModuleDict(_UpperCamelCase				)






   def 	A__  (  self:			int   ,lowerCamelCase_:			Any				)     ->	str:
       return get_trunk_forward_outputs(
           _UpperCamelCase   ,out_feat_keys=_UpperCamelCase   ,feature_blocks=self._feature_blocks   ,)







class 	_snake_case							(			__a      ):






   '''simple docstring'''






   def 	A__  (  self:			str   ,lowerCamelCase_:			Optional[Any]				)     ->	str:
       UpperCAmelCase_			:    Optional[int]   						= x.split("""-"""				)
       return x_split[0] + x_split[1] + "_" + "".join(x_split[2:]				)






   def __getitem__(  self:			int   ,lowerCamelCase_:			Dict				)     ->	Callable[[], Tuple[nn.Module, Dict]]:
       # default to timm!
       if x not in self:
           UpperCAmelCase_			:    Union[str, Any]   						= self.convert_name_to_timm(_UpperCamelCase				)
           UpperCAmelCase_			:    List[str]   						= partial(lambda: (timm.create_model(_UpperCamelCase   ,pretrained=_UpperCamelCase				).eval(), None)				)

       else:
           UpperCAmelCase_			:    str   						= super().__getitem__(_UpperCamelCase				)

       return val







class 	_snake_case							(			__a      ):






   '''simple docstring'''






   def __getitem__(  self:			Tuple   ,lowerCamelCase_:			Union[str, Any]				)     ->	Callable[[], nn.Module]:
       if "seer" in x and "in1k" not in x:
           UpperCAmelCase_			:    str   						= RegNetModel
       else:
           UpperCAmelCase_			:    Tuple   						= RegNetForImageClassification
       return val







def 	lowerCamelCase_   (  _a    :		Union[str, Any]					,						_a    :		Optional[Any]					,						_a    :		List[Tuple[str, str]]  ):



    '''simple docstring'''




    for from_key, to_key in keys:
        UpperCAmelCase_			:    Optional[Any]   						= from_state_dict[from_key].clone()
        print(F'''Copied key={from_key} to={to_key}'''  )
    return to_state_dict







def 	lowerCamelCase_   (  _a    :		str					,						_a    :		Callable[[], nn.Module]					,						_a    :		Callable[[], nn.Module]					,						_a    :		RegNetConfig					,						_a    :		Path					,						_a    :		bool = True					,						):



    '''simple docstring'''




    print(F'''Converting {name}...'''  )
    with torch.no_grad():
        UpperCAmelCase_						,      UpperCAmelCase_			:    Optional[int]   						= from_model_func()
        UpperCAmelCase_			:    Tuple   						= our_model_func(__lowerCAmelCase  ).eval()
        UpperCAmelCase_			:    List[str]   						= ModuleTransfer(src=__lowerCAmelCase					,						dest=__lowerCAmelCase					,						raise_if_mismatch=__lowerCAmelCase  )
        UpperCAmelCase_			:    Optional[int]   						= torch.randn((1, 3, 224, 224)  )
        module_transfer(__lowerCAmelCase  )

    if from_state_dict is not None:
        UpperCAmelCase_			:    int   						= []
        # for seer - in1k finetuned we have to manually copy the head
        if "seer" in name and "in1k" in name:
            UpperCAmelCase_			:    List[str]   						= [("""0.clf.0.weight""", """classifier.1.weight"""), ("""0.clf.0.bias""", """classifier.1.bias""")]
        UpperCAmelCase_			:    str   						= manually_copy_vissl_head(__lowerCAmelCase					,						our_model.state_dict()					,						__lowerCAmelCase  )
        our_model.load_state_dict(__lowerCAmelCase  )

    UpperCAmelCase_			:    Optional[Any]   						= our_model(__lowerCAmelCase					,						output_hidden_states=__lowerCAmelCase  )
    UpperCAmelCase_			:    List[str]   						= (
        our_outputs.logits if isinstance(__lowerCAmelCase					,						__lowerCAmelCase  ) else our_outputs.last_hidden_state
    )

    UpperCAmelCase_			:    Optional[int]   						= from_model(__lowerCAmelCase  )
    UpperCAmelCase_			:    int   						= from_output[-1] if type(__lowerCAmelCase  ) is list else from_output

    # now since I don't want to use any config files, vissl seer model doesn't actually have an head, so let's just check the last hidden state
    if "seer" in name and "in1k" in name:
        UpperCAmelCase_			:    Union[str, Any]   						= our_outputs.hidden_states[-1]

    assert torch.allclose(__lowerCAmelCase					,						__lowerCAmelCase  ), "The model logits don't match the original one."

    if push_to_hub:
        our_model.push_to_hub(
            repo_path_or_name=save_directory / name					,						commit_message="""Add model"""					,						use_temp_dir=__lowerCAmelCase					,						)

        UpperCAmelCase_			:    List[Any]   						= 224 if """seer""" not in name else 384
        # we can use the convnext one
        UpperCAmelCase_			:    Any   						= AutoImageProcessor.from_pretrained("""facebook/convnext-base-224-22k-1k"""					,						size=__lowerCAmelCase  )
        image_processor.push_to_hub(
            repo_path_or_name=save_directory / name					,						commit_message="""Add image processor"""					,						use_temp_dir=__lowerCAmelCase					,						)

        print(F'''Pushed {name}'''  )







def 	lowerCamelCase_   (  _a    :		Path					,						_a    :		str = None					,						_a    :		bool = True  ):



    '''simple docstring'''




    UpperCAmelCase_			:    Optional[Any]   						= """imagenet-1k-id2label.json"""
    UpperCAmelCase_			:    int   						= 1000
    UpperCAmelCase_			:    Optional[Any]   						= (1, num_labels)

    UpperCAmelCase_			:    List[Any]   						= """huggingface/label-files"""
    UpperCAmelCase_			:    int   						= num_labels
    UpperCAmelCase_			:    Union[str, Any]   						= json.load(open(cached_download(hf_hub_url(__lowerCAmelCase					,						__lowerCAmelCase					,						repo_type="""dataset"""  )  )					,						"""r"""  )  )
    UpperCAmelCase_			:    Optional[Any]   						= {int(__lowerCAmelCase  ): v for k, v in idalabel.items()}

    UpperCAmelCase_			:    int   						= idalabel
    UpperCAmelCase_			:    Any   						= {v: k for k, v in idalabel.items()}

    UpperCAmelCase_			:    Optional[Any]   						= partial(__lowerCAmelCase					,						num_labels=__lowerCAmelCase					,						idalabel=__lowerCAmelCase					,						labelaid=__lowerCAmelCase  )

    UpperCAmelCase_			:    Dict   						= {
        """regnet-x-002""": ImageNetPreTrainedConfig(
            depths=[1, 1, 4, 7]					,						hidden_sizes=[24, 56, 152, 368]					,						groups_width=8					,						layer_type="""x"""  ),
        """regnet-x-004""": ImageNetPreTrainedConfig(
            depths=[1, 2, 7, 12]					,						hidden_sizes=[32, 64, 160, 384]					,						groups_width=16					,						layer_type="""x"""  ),
        """regnet-x-006""": ImageNetPreTrainedConfig(
            depths=[1, 3, 5, 7]					,						hidden_sizes=[48, 96, 240, 528]					,						groups_width=24					,						layer_type="""x"""  ),
        """regnet-x-008""": ImageNetPreTrainedConfig(
            depths=[1, 3, 7, 5]					,						hidden_sizes=[64, 128, 288, 672]					,						groups_width=16					,						layer_type="""x"""  ),
        """regnet-x-016""": ImageNetPreTrainedConfig(
            depths=[2, 4, 10, 2]					,						hidden_sizes=[72, 168, 408, 912]					,						groups_width=24					,						layer_type="""x"""  ),
        """regnet-x-032""": ImageNetPreTrainedConfig(
            depths=[2, 6, 15, 2]					,						hidden_sizes=[96, 192, 432, 1008]					,						groups_width=48					,						layer_type="""x"""  ),
        """regnet-x-040""": ImageNetPreTrainedConfig(
            depths=[2, 5, 14, 2]					,						hidden_sizes=[80, 240, 560, 1360]					,						groups_width=40					,						layer_type="""x"""  ),
        """regnet-x-064""": ImageNetPreTrainedConfig(
            depths=[2, 4, 10, 1]					,						hidden_sizes=[168, 392, 784, 1624]					,						groups_width=56					,						layer_type="""x"""  ),
        """regnet-x-080""": ImageNetPreTrainedConfig(
            depths=[2, 5, 15, 1]					,						hidden_sizes=[80, 240, 720, 1920]					,						groups_width=120					,						layer_type="""x"""  ),
        """regnet-x-120""": ImageNetPreTrainedConfig(
            depths=[2, 5, 11, 1]					,						hidden_sizes=[224, 448, 896, 2240]					,						groups_width=112					,						layer_type="""x"""  ),
        """regnet-x-160""": ImageNetPreTrainedConfig(
            depths=[2, 6, 13, 1]					,						hidden_sizes=[256, 512, 896, 2048]					,						groups_width=128					,						layer_type="""x"""  ),
        """regnet-x-320""": ImageNetPreTrainedConfig(
            depths=[2, 7, 13, 1]					,						hidden_sizes=[336, 672, 1344, 2520]					,						groups_width=168					,						layer_type="""x"""  ),
        # y variant
        """regnet-y-002""": ImageNetPreTrainedConfig(depths=[1, 1, 4, 7]					,						hidden_sizes=[24, 56, 152, 368]					,						groups_width=8  ),
        """regnet-y-004""": ImageNetPreTrainedConfig(
            depths=[1, 3, 6, 6]					,						hidden_sizes=[48, 104, 208, 440]					,						groups_width=8  ),
        """regnet-y-006""": ImageNetPreTrainedConfig(
            depths=[1, 3, 7, 4]					,						hidden_sizes=[48, 112, 256, 608]					,						groups_width=16  ),
        """regnet-y-008""": ImageNetPreTrainedConfig(
            depths=[1, 3, 8, 2]					,						hidden_sizes=[64, 128, 320, 768]					,						groups_width=16  ),
        """regnet-y-016""": ImageNetPreTrainedConfig(
            depths=[2, 6, 17, 2]					,						hidden_sizes=[48, 120, 336, 888]					,						groups_width=24  ),
        """regnet-y-032""": ImageNetPreTrainedConfig(
            depths=[2, 5, 13, 1]					,						hidden_sizes=[72, 216, 576, 1512]					,						groups_width=24  ),
        """regnet-y-040""": ImageNetPreTrainedConfig(
            depths=[2, 6, 12, 2]					,						hidden_sizes=[128, 192, 512, 1088]					,						groups_width=64  ),
        """regnet-y-064""": ImageNetPreTrainedConfig(
            depths=[2, 7, 14, 2]					,						hidden_sizes=[144, 288, 576, 1296]					,						groups_width=72  ),
        """regnet-y-080""": ImageNetPreTrainedConfig(
            depths=[2, 4, 10, 1]					,						hidden_sizes=[168, 448, 896, 2016]					,						groups_width=56  ),
        """regnet-y-120""": ImageNetPreTrainedConfig(
            depths=[2, 5, 11, 1]					,						hidden_sizes=[224, 448, 896, 2240]					,						groups_width=112  ),
        """regnet-y-160""": ImageNetPreTrainedConfig(
            depths=[2, 4, 11, 1]					,						hidden_sizes=[224, 448, 1232, 3024]					,						groups_width=112  ),
        """regnet-y-320""": ImageNetPreTrainedConfig(
            depths=[2, 5, 12, 1]					,						hidden_sizes=[232, 696, 1392, 3712]					,						groups_width=232  ),
        # models created by SEER -> https://arxiv.org/abs/2202.08360
        """regnet-y-320-seer""": RegNetConfig(depths=[2, 5, 12, 1]					,						hidden_sizes=[232, 696, 1392, 3712]					,						groups_width=232  ),
        """regnet-y-640-seer""": RegNetConfig(depths=[2, 5, 12, 1]					,						hidden_sizes=[328, 984, 1968, 4920]					,						groups_width=328  ),
        """regnet-y-1280-seer""": RegNetConfig(
            depths=[2, 7, 17, 1]					,						hidden_sizes=[528, 1056, 2904, 7392]					,						groups_width=264  ),
        """regnet-y-2560-seer""": RegNetConfig(
            depths=[3, 7, 16, 1]					,						hidden_sizes=[640, 1696, 2544, 5088]					,						groups_width=640  ),
        """regnet-y-10b-seer""": ImageNetPreTrainedConfig(
            depths=[2, 7, 17, 1]					,						hidden_sizes=[2020, 4040, 1_1110, 2_8280]					,						groups_width=1010  ),
        # finetuned on imagenet
        """regnet-y-320-seer-in1k""": ImageNetPreTrainedConfig(
            depths=[2, 5, 12, 1]					,						hidden_sizes=[232, 696, 1392, 3712]					,						groups_width=232  ),
        """regnet-y-640-seer-in1k""": ImageNetPreTrainedConfig(
            depths=[2, 5, 12, 1]					,						hidden_sizes=[328, 984, 1968, 4920]					,						groups_width=328  ),
        """regnet-y-1280-seer-in1k""": ImageNetPreTrainedConfig(
            depths=[2, 7, 17, 1]					,						hidden_sizes=[528, 1056, 2904, 7392]					,						groups_width=264  ),
        """regnet-y-2560-seer-in1k""": ImageNetPreTrainedConfig(
            depths=[3, 7, 16, 1]					,						hidden_sizes=[640, 1696, 2544, 5088]					,						groups_width=640  ),
        """regnet-y-10b-seer-in1k""": ImageNetPreTrainedConfig(
            depths=[2, 7, 17, 1]					,						hidden_sizes=[2020, 4040, 1_1110, 2_8280]					,						groups_width=1010  ),
    }

    UpperCAmelCase_			:    str   						= NameToOurModelFuncMap()
    UpperCAmelCase_			:    Any   						= NameToFromModelFuncMap()
    # add seer weights logic

    def load_using_classy_vision(_a    :		str					,						_a    :		Callable[[], nn.Module]  ) -> Tuple[nn.Module, Dict]:
        UpperCAmelCase_			:    int   						= torch.hub.load_state_dict_from_url(__lowerCAmelCase					,						model_dir=str(__lowerCAmelCase  )					,						map_location="""cpu"""  )
        UpperCAmelCase_			:    Any   						= model_func()
        # check if we have a head, if yes add it
        UpperCAmelCase_			:    List[str]   						= files["""classy_state_dict"""]["""base_model"""]["""model"""]
        UpperCAmelCase_			:    str   						= model_state_dict["""trunk"""]
        model.load_state_dict(__lowerCAmelCase  )
        return model.eval(), model_state_dict["heads"]

    # pretrained
    UpperCAmelCase_			:    Tuple   						= partial(
        __lowerCAmelCase					,						"""https://dl.fbaipublicfiles.com/vissl/model_zoo/seer_regnet32d/seer_regnet32gf_model_iteration244000.torch"""					,						lambda: FakeRegNetVisslWrapper(RegNetYaagf()  )					,						)

    UpperCAmelCase_			:    Any   						= partial(
        __lowerCAmelCase					,						"""https://dl.fbaipublicfiles.com/vissl/model_zoo/seer_regnet64/seer_regnet64gf_model_final_checkpoint_phase0.torch"""					,						lambda: FakeRegNetVisslWrapper(RegNetYaagf()  )					,						)

    UpperCAmelCase_			:    List[str]   						= partial(
        __lowerCAmelCase					,						"""https://dl.fbaipublicfiles.com/vissl/model_zoo/swav_ig1b_regnet128Gf_cnstant_bs32_node16_sinkhorn10_proto16k_syncBN64_warmup8k/model_final_checkpoint_phase0.torch"""					,						lambda: FakeRegNetVisslWrapper(RegNetYaaagf()  )					,						)

    UpperCAmelCase_			:    Dict   						= partial(
        __lowerCAmelCase					,						"""https://dl.fbaipublicfiles.com/vissl/model_zoo/seer_regnet10B/model_iteration124500_conso.torch"""					,						lambda: FakeRegNetVisslWrapper(
            RegNet(RegNetParams(depth=27					,						group_width=1010					,						w_a=1744					,						w_a=6_2_0.8_3					,						w_m=2.5_2  )  )  )					,						)

    # IN1K finetuned
    UpperCAmelCase_			:    Optional[int]   						= partial(
        __lowerCAmelCase					,						"""https://dl.fbaipublicfiles.com/vissl/model_zoo/seer_finetuned/seer_regnet32_finetuned_in1k_model_final_checkpoint_phase78.torch"""					,						lambda: FakeRegNetVisslWrapper(RegNetYaagf()  )					,						)

    UpperCAmelCase_			:    str   						= partial(
        __lowerCAmelCase					,						"""https://dl.fbaipublicfiles.com/vissl/model_zoo/seer_finetuned/seer_regnet64_finetuned_in1k_model_final_checkpoint_phase78.torch"""					,						lambda: FakeRegNetVisslWrapper(RegNetYaagf()  )					,						)

    UpperCAmelCase_			:    Union[str, Any]   						= partial(
        __lowerCAmelCase					,						"""https://dl.fbaipublicfiles.com/vissl/model_zoo/seer_finetuned/seer_regnet128_finetuned_in1k_model_final_checkpoint_phase78.torch"""					,						lambda: FakeRegNetVisslWrapper(RegNetYaaagf()  )					,						)

    UpperCAmelCase_			:    Optional[Any]   						= partial(
        __lowerCAmelCase					,						"""https://dl.fbaipublicfiles.com/vissl/model_zoo/seer_finetuned/seer_10b_finetuned_in1k_model_phase28_conso.torch"""					,						lambda: FakeRegNetVisslWrapper(
            RegNet(RegNetParams(depth=27					,						group_width=1010					,						w_a=1744					,						w_a=6_2_0.8_3					,						w_m=2.5_2  )  )  )					,						)

    if model_name:
        convert_weight_and_push(
            __lowerCAmelCase					,						names_to_from_model_map[model_name]					,						names_to_ours_model_map[model_name]					,						names_to_config[model_name]					,						__lowerCAmelCase					,						__lowerCAmelCase					,						)
    else:
        for model_name, config in names_to_config.items():
            convert_weight_and_push(
                __lowerCAmelCase					,						names_to_from_model_map[model_name]					,						names_to_ours_model_map[model_name]					,						__lowerCAmelCase					,						__lowerCAmelCase					,						__lowerCAmelCase					,						)
    return config, expected_shape


if __name__ == "__main__":
      UpperCamelCase_										=					argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          '''--model_name''',
          default=None,
          type=str,
          help=(
              '''The name of the model you wish to convert, it must be one of the supported regnet* architecture,'''
              ''' currently: regnetx-*, regnety-*. If `None`, all of them will the converted.'''
          ),
      )
      parser.add_argument(
          '''--pytorch_dump_folder_path''',
          default=None,
          type=Path,
          required=True,
          help='''Path to the output PyTorch model directory.''',
      )
      parser.add_argument(
          '''--push_to_hub''',
          default=True,
          type=bool,
          required=False,
          help='''If True, push model and image processor to the hub.''',
      )

      UpperCamelCase_										=					parser.parse_args()

      UpperCamelCase_										=					args.pytorch_dump_folder_path
      pytorch_dump_folder_path.mkdir(exist_ok=True, parents=True)
      convert_weights_and_push(pytorch_dump_folder_path, args.model_name, args.push_to_hub)


import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 	_lowerCAmelCase					(	__a					,      unittest.TestCase       ):
				_lowercase					=CLIPTokenizer
				_lowercase					=CLIPTokenizerFast
				_lowercase					=True
				_lowercase					={}
				_lowercase					=False




				def      __a						(					self     )				->      Dict:
							super().setUp()

							# fmt: off
							lowerCAmelCase_						=	["l", "o", "w", "e", "r", "s", "t", "i", "d", "n", "lo", "l</w>", "w</w>", "r</w>", "t</w>", "low</w>", "er</w>", "lowest</w>", "newer</w>", "wider", "<unk>", "<|startoftext|>", "<|endoftext|>"]
							# fmt: on
							lowerCAmelCase_						=	dict(zip(_UpperCamelCase				,				range(len(_UpperCamelCase     )     )     )     )
							lowerCAmelCase_						=	["#version: 0.2", "l o", "lo w</w>", "e r</w>"]
							lowerCAmelCase_						=	{"unk_token": "<unk>"}

							lowerCAmelCase_						=	os.path.join(self.tmpdirname				,				VOCAB_FILES_NAMES["vocab_file"]     )
							lowerCAmelCase_						=	os.path.join(self.tmpdirname				,				VOCAB_FILES_NAMES["merges_file"]     )
							with open(self.vocab_file				,				"w"				,				encoding="utf-8"     ) as fp:
										fp.write(json.dumps(_UpperCamelCase     ) + "\n"     )
							with open(self.merges_file				,				"w"				,				encoding="utf-8"     ) as fp:
										fp.write("\n".join(_UpperCamelCase     )     )




				def      __a						(					self				,				**_UpperCamelCase     )				->      Any:
							kwargs.update(self.special_tokens_map     )
							return CLIPTokenizer.from_pretrained(self.tmpdirname				,				**_UpperCamelCase     )




				def      __a						(					self				,				**_UpperCamelCase     )				->      int:
							kwargs.update(self.special_tokens_map     )
							return CLIPTokenizerFast.from_pretrained(self.tmpdirname				,				**_UpperCamelCase     )




				def      __a						(					self				,				_UpperCamelCase     )				->      List[str]:
							lowerCAmelCase_						=	"lower newer"
							lowerCAmelCase_						=	"lower newer"
							return input_text, output_text




				def      __a						(					self     )				->      List[Any]:
							lowerCAmelCase_						=	CLIPTokenizer(self.vocab_file				,				self.merges_file				,				**self.special_tokens_map     )
							lowerCAmelCase_						=	"lower newer"
							lowerCAmelCase_						=	["lo", "w", "er</w>", "n", "e", "w", "er</w>"]
							lowerCAmelCase_						=	tokenizer.tokenize(_UpperCamelCase     )
							self.assertListEqual(_UpperCamelCase				,				_UpperCamelCase     )

							lowerCAmelCase_						=	tokens + [tokenizer.unk_token]
							lowerCAmelCase_						=	[10, 2, 16, 9, 3, 2, 16, 20]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase     )				,				_UpperCamelCase     )




				@require_ftfy
				def      __a						(					self     )				->      Union[str, Any]:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
													lowerCAmelCase_						=	self.tokenizer_class.from_pretrained(_UpperCamelCase				,				**_UpperCamelCase     )
													lowerCAmelCase_						=	self.rust_tokenizer_class.from_pretrained(_UpperCamelCase				,				**_UpperCamelCase     )

													lowerCAmelCase_						=	"A\n'll 11p223RF☆ho!!to?'d'd''d of a cat to-$''d."
													lowerCAmelCase_						=	tokenizer_s.tokenize(_UpperCamelCase     )
													lowerCAmelCase_						=	tokenizer_r.tokenize(_UpperCamelCase     )

													self.assertListEqual(_UpperCamelCase				,				_UpperCamelCase     )

													# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
													# with Tilde) encoded in 2 different ways
													lowerCAmelCase_						=	"xa\u0303y" + " " + "x\xe3y"
													lowerCAmelCase_						=	tokenizer_s.tokenize(_UpperCamelCase     )
													lowerCAmelCase_						=	tokenizer_r.tokenize(_UpperCamelCase     )

													self.assertListEqual(_UpperCamelCase				,				_UpperCamelCase     )

													# Test that the tokenization is identical on unicode of space type
													lowerCAmelCase_						=	[
													    "\u0009",  # (horizontal tab, '\t')
													    "\u000B",  # (vertical tab)
													    "\u000C",  # (form feed)
													    "\u0020",  # (space, ' ')
													    "\u200E",  # (left-to-right mark):w
													    "\u200F",  # (right-to-left mark)
													]
													for unicode_seq in spaces_unicodes:
																lowerCAmelCase_						=	tokenizer_s.tokenize(_UpperCamelCase     )
																lowerCAmelCase_						=	tokenizer_r.tokenize(_UpperCamelCase     )

																self.assertListEqual(_UpperCamelCase				,				_UpperCamelCase     )

													# Test that the tokenization is identical on unicode of line break type
													lowerCAmelCase_						=	[
													    "\u000A",  # (line feed, '\n')
													    "\r\n",  # (carriage return and line feed, '\r\n')
													    "\u000D",  # (carriage return, '\r')
													    "\r",  # (carriage return, '\r')
													    "\u000D",  # (carriage return, '\r')
													    "\u2028",  # (line separator)
													    "\u2029",  # (paragraph separator)
													    # "\u0085", # (next line)
													]

													# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
													# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
													# space (and thus into an empty list).

													for unicode_seq in line_break_unicodes:
																lowerCAmelCase_						=	tokenizer_s.tokenize(_UpperCamelCase     )
																lowerCAmelCase_						=	tokenizer_r.tokenize(_UpperCamelCase     )

																self.assertListEqual(_UpperCamelCase				,				_UpperCamelCase     )




				def      __a						(					self     )				->      str:
							# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
													lowerCAmelCase_						=	"hello"  # `hello` is a token in the vocabulary of `pretrained_name`
													lowerCAmelCase_						=	f"""{text_of_1_token} {text_of_1_token}"""

													lowerCAmelCase_						=	self.rust_tokenizer_class.from_pretrained(
													    _UpperCamelCase				,				use_fast=_UpperCamelCase				,				)
													lowerCAmelCase_						=	tokenizer_r(_UpperCamelCase				,				return_offsets_mapping=_UpperCamelCase				,				add_special_tokens=_UpperCamelCase     )
													self.assertEqual(encoding.offset_mapping[0]				,				(0, len(_UpperCamelCase     ))     )
													self.assertEqual(
													    encoding.offset_mapping[1]				,				(len(_UpperCamelCase     ) + 1, len(_UpperCamelCase     ) + 1 + len(_UpperCamelCase     ))				,				)

													lowerCAmelCase_						=	f""" {text}"""

													lowerCAmelCase_						=	self.rust_tokenizer_class.from_pretrained(
													    _UpperCamelCase				,				use_fast=_UpperCamelCase				,				)
													lowerCAmelCase_						=	tokenizer_r(_UpperCamelCase				,				return_offsets_mapping=_UpperCamelCase				,				add_special_tokens=_UpperCamelCase     )
													self.assertEqual(encoding.offset_mapping[0]				,				(1, 1 + len(_UpperCamelCase     ))     )
													self.assertEqual(
													    encoding.offset_mapping[1]				,				(1 + len(_UpperCamelCase     ) + 1, 1 + len(_UpperCamelCase     ) + 1 + len(_UpperCamelCase     ))				,				)




				def      __a						(					self     )				->      Optional[Any]:
							# Test related to the breaking change introduced in transformers v4.17.0
							# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
							with self.assertRaises(_UpperCamelCase     ) as context:
										self.rust_tokenizer_class.from_pretrained("robot-test/old-clip-tokenizer"     )

							self.assertTrue(
							    context.exception.args[0].startswith(
							        "The `backend_tokenizer` provided does not match the expected format."     )     )




				@require_ftfy
				def      __a						(					self     )				->      str:
							super().test_tokenization_python_rust_equals()




				def      __a						(					self     )				->      Any:
							# CLIP always lower cases letters
							pass



