



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__A   			=logging.get_logger(__name__)

__A   			={"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A   			={
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

__A   			={
    "distilbert-base-uncased": 5_1_2,
    "distilbert-base-uncased-distilled-squad": 5_1_2,
    "distilbert-base-cased": 5_1_2,
    "distilbert-base-cased-distilled-squad": 5_1_2,
    "distilbert-base-german-cased": 5_1_2,
    "distilbert-base-multilingual-cased": 5_1_2,
}


__A   			={
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}

class 		UpperCAmelCase__			(			__UpperCamelCase   ):




			'''simple docstring'''






			UpperCamelCase									=       VOCAB_FILES_NAMES
			UpperCamelCase									=       PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase									=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase									=       PRETRAINED_INIT_CONFIGURATION
			UpperCamelCase									=       ["""input_ids""", """attention_mask"""]
			UpperCamelCase									=       DistilBertTokenizer




			def __init__(   self							: Any       ,       a_							: Union[str, Any]=None       ,       a_							: Tuple=None       ,       a_							: List[str]=True       ,       a_							: List[str]="[UNK]"       ,       a_							: Union[str, Any]="[SEP]"       ,       a_							: Tuple="[PAD]"       ,       a_							: Union[str, Any]="[CLS]"       ,       a_							: Dict="[MASK]"       ,       a_							: str=True       ,       a_							: int=None       ,       **a_							: List[str]       ,       ):
					'''simple docstring'''





					super().__init__(
					    a_       ,       tokenizer_file=a_       ,       do_lower_case=a_       ,       unk_token=a_       ,       sep_token=a_       ,       pad_token=a_       ,       cls_token=a_       ,       mask_token=a_       ,       tokenize_chinese_chars=a_       ,       strip_accents=a_       ,       **a_       ,       )

					__UpperCAmelCase    :     Optional[Any]													=	json.loads(self.backend_tokenizer.normalizer.__getstate__() )
					if (
					    normalizer_state.get('''lowercase'''       ,       a_ ) != do_lower_case
					    or normalizer_state.get('''strip_accents'''       ,       a_ ) != strip_accents
					    or normalizer_state.get('''handle_chinese_chars'''       ,       a_ ) != tokenize_chinese_chars
					):
							__UpperCAmelCase    :     int													=	getattr(a_       ,       normalizer_state.pop('''type''' ) )
							__UpperCAmelCase    :     int													=	do_lower_case
							__UpperCAmelCase    :     Union[str, Any]													=	strip_accents
							__UpperCAmelCase    :     Any													=	tokenize_chinese_chars
							__UpperCAmelCase    :     int													=	normalizer_class(**a_ )

					__UpperCAmelCase    :     List[str]													=	do_lower_case




			def  snake_case__     (   self							: Tuple       ,       a_							: Tuple       ,       a_							: Optional[int]=None ):
					'''simple docstring'''





					__UpperCAmelCase    :     Dict													=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

					return output




			def  snake_case__     (   self							: Union[str, Any]       ,       a_							: List[int]       ,       a_							: Optional[List[int]] = None ):
					'''simple docstring'''





					__UpperCAmelCase    :     Tuple													=	[self.sep_token_id]
					__UpperCAmelCase    :     Any													=	[self.cls_token_id]
					if token_ids_a is None:
							return len(cls + token_ids_a + sep ) * [0]
					return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]




			def  snake_case__     (   self							: Tuple       ,       a_							: str       ,       a_							: Optional[str] = None ):
					'''simple docstring'''





					__UpperCAmelCase    :     List[str]													=	self._tokenizer.model.save(a_       ,       name=a_ )
					return tuple(a_ )










from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


__A   			=logging.get_logger(__name__)


if is_vision_available():
				import PIL

class 		UpperCAmelCase__			(			__UpperCamelCase   ):




			'''simple docstring'''






			UpperCamelCase									=       ["""pixel_values"""]




			def __init__(   self							: Tuple       ,       a_							: bool = True       ,       a_							: Dict[str, int] = None       ,       a_							: PILImageResampling = PILImageResampling.BICUBIC       ,       a_							: bool = True       ,       a_							: Dict[str, int] = None       ,       a_							: bool = True       ,       a_							: Union[int, float] = 1 / 2_55       ,       a_							: bool = True       ,       a_							: Optional[Union[float, List[float]]] = None       ,       a_							: Optional[Union[float, List[float]]] = None       ,       a_							: bool = True       ,       **a_							: str       ,       ):
					'''simple docstring'''





					super().__init__(**a_ )
					__UpperCAmelCase    :     List[Any]													=	size if size is not None else {'''shortest_edge''': 2_24}
					__UpperCAmelCase    :     List[str]													=	get_size_dict(a_       ,       default_to_square=a_ )
					__UpperCAmelCase    :     int													=	crop_size if crop_size is not None else {'''height''': 2_24, '''width''': 2_24}
					__UpperCAmelCase    :     Optional[int]													=	get_size_dict(a_       ,       default_to_square=a_       ,       param_name='''crop_size''' )

					__UpperCAmelCase    :     int													=	do_resize
					__UpperCAmelCase    :     Union[str, Any]													=	size
					__UpperCAmelCase    :     Union[str, Any]													=	resample
					__UpperCAmelCase    :     Any													=	do_center_crop
					__UpperCAmelCase    :     Any													=	crop_size
					__UpperCAmelCase    :     Any													=	do_rescale
					__UpperCAmelCase    :     Dict													=	rescale_factor
					__UpperCAmelCase    :     Union[str, Any]													=	do_normalize
					__UpperCAmelCase    :     Any													=	image_mean if image_mean is not None else OPENAI_CLIP_MEAN
					__UpperCAmelCase    :     int													=	image_std if image_std is not None else OPENAI_CLIP_STD
					__UpperCAmelCase    :     List[str]													=	do_convert_rgb




			def  snake_case__     (   self							: Optional[Any]       ,       a_							: np.ndarray       ,       a_							: Dict[str, int]       ,       a_							: PILImageResampling = PILImageResampling.BICUBIC       ,       a_							: Optional[Union[str, ChannelDimension]] = None       ,       **a_							: Optional[int]       ,       ):
					'''simple docstring'''





					__UpperCAmelCase    :     Dict													=	get_size_dict(a_       ,       default_to_square=a_ )
					if "shortest_edge" not in size:
							raise ValueError(F'The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}' )
					__UpperCAmelCase    :     Optional[int]													=	get_resize_output_image_size(a_       ,       size=size['''shortest_edge''']       ,       default_to_square=a_ )
					return resize(a_       ,       size=a_       ,       resample=a_       ,       data_format=a_       ,       **a_ )




			def  snake_case__     (   self							: Union[str, Any]       ,       a_							: np.ndarray       ,       a_							: Dict[str, int]       ,       a_							: Optional[Union[str, ChannelDimension]] = None       ,       **a_							: Dict       ,       ):
					'''simple docstring'''





					__UpperCAmelCase    :     Optional[Any]													=	get_size_dict(a_ )
					if "height" not in size or "width" not in size:
							raise ValueError(F'The `size` parameter must contain the keys (height, width). Got {size.keys()}' )
					return center_crop(a_       ,       size=(size['''height'''], size['''width'''])       ,       data_format=a_       ,       **a_ )




			def  snake_case__     (   self							: Union[str, Any]       ,       a_							: np.ndarray       ,       a_							: Union[int, float]       ,       a_							: Optional[Union[str, ChannelDimension]] = None       ,       **a_							: List[str]       ,       ):
					'''simple docstring'''





					return rescale(a_       ,       scale=a_       ,       data_format=a_       ,       **a_ )




			def  snake_case__     (   self							: Optional[Any]       ,       a_							: np.ndarray       ,       a_							: Union[float, List[float]]       ,       a_							: Union[float, List[float]]       ,       a_							: Optional[Union[str, ChannelDimension]] = None       ,       **a_							: Dict       ,       ):
					'''simple docstring'''





					return normalize(a_       ,       mean=a_       ,       std=a_       ,       data_format=a_       ,       **a_ )




			def  snake_case__     (   self							: Any       ,       a_							: ImageInput       ,       a_							: bool = None       ,       a_							: Dict[str, int] = None       ,       a_							: PILImageResampling = None       ,       a_							: bool = None       ,       a_							: int = None       ,       a_							: bool = None       ,       a_							: float = None       ,       a_							: bool = None       ,       a_							: Optional[Union[float, List[float]]] = None       ,       a_							: Optional[Union[float, List[float]]] = None       ,       a_							: bool = None       ,       a_							: Optional[Union[str, TensorType]] = None       ,       a_							: Optional[ChannelDimension] = ChannelDimension.FIRST       ,       **a_							: Dict       ,       ):
					'''simple docstring'''





					__UpperCAmelCase    :     Tuple													=	do_resize if do_resize is not None else self.do_resize
					__UpperCAmelCase    :     Optional[Any]													=	size if size is not None else self.size
					__UpperCAmelCase    :     Dict													=	get_size_dict(a_       ,       param_name='''size'''       ,       default_to_square=a_ )
					__UpperCAmelCase    :     int													=	resample if resample is not None else self.resample
					__UpperCAmelCase    :     Optional[Any]													=	do_center_crop if do_center_crop is not None else self.do_center_crop
					__UpperCAmelCase    :     Any													=	crop_size if crop_size is not None else self.crop_size
					__UpperCAmelCase    :     Dict													=	get_size_dict(a_       ,       param_name='''crop_size'''       ,       default_to_square=a_ )
					__UpperCAmelCase    :     int													=	do_rescale if do_rescale is not None else self.do_rescale
					__UpperCAmelCase    :     Optional[int]													=	rescale_factor if rescale_factor is not None else self.rescale_factor
					__UpperCAmelCase    :     List[str]													=	do_normalize if do_normalize is not None else self.do_normalize
					__UpperCAmelCase    :     Optional[int]													=	image_mean if image_mean is not None else self.image_mean
					__UpperCAmelCase    :     Tuple													=	image_std if image_std is not None else self.image_std
					__UpperCAmelCase    :     List[str]													=	do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

					__UpperCAmelCase    :     List[str]													=	make_list_of_images(a_ )

					if not valid_images(a_ ):
							raise ValueError(
							    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
							    '''torch.Tensor, tf.Tensor or jax.ndarray.''' )

					if do_resize and size is None:
							raise ValueError('''Size must be specified if do_resize is True.''' )

					if do_center_crop and crop_size is None:
							raise ValueError('''Crop size must be specified if do_center_crop is True.''' )

					if do_rescale and rescale_factor is None:
							raise ValueError('''Rescale factor must be specified if do_rescale is True.''' )

					if do_normalize and (image_mean is None or image_std is None):
							raise ValueError('''Image mean and std must be specified if do_normalize is True.''' )

					# PIL RGBA images are converted to RGB
					if do_convert_rgb:
							__UpperCAmelCase    :     Optional[Any]													=	[convert_to_rgb(a_ ) for image in images]

					# All transformations expect numpy arrays.
					__UpperCAmelCase    :     int													=	[to_numpy_array(a_ ) for image in images]

					if do_resize:
							__UpperCAmelCase    :     int													=	[self.resize(image=a_       ,       size=a_       ,       resample=a_ ) for image in images]

					if do_center_crop:
							__UpperCAmelCase    :     List[str]													=	[self.center_crop(image=a_       ,       size=a_ ) for image in images]

					if do_rescale:
							__UpperCAmelCase    :     Dict													=	[self.rescale(image=a_       ,       scale=a_ ) for image in images]

					if do_normalize:
							__UpperCAmelCase    :     Optional[int]													=	[self.normalize(image=a_       ,       mean=a_       ,       std=a_ ) for image in images]

					__UpperCAmelCase    :     Optional[int]													=	[to_channel_dimension_format(a_       ,       a_ ) for image in images]

					__UpperCAmelCase    :     Union[str, Any]													=	{'''pixel_values''': images}
					return BatchFeature(data=a_       ,       tensor_type=a_ )





