



"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


UpperCAmelCase  :						str	=       logging.get_logger(__name__)

UpperCAmelCase  :						Optional[int]	=       {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase  :						Tuple	=       {
    """vocab_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/vocab.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/vocab.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/vocab.json""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/merges.txt""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/merges.txt""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/merges.txt""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/tokenizer.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/tokenizer.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json""",
        """roberta-base-openai-detector""": (
            """https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"""
        ),
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCAmelCase  :						List[Any]	=       {
    """roberta-base""": 512,
    """roberta-large""": 512,
    """roberta-large-mnli""": 512,
    """distilroberta-base""": 512,
    """roberta-base-openai-detector""": 512,
    """roberta-large-openai-detector""": 512,
}


class       SCREAMING_SNAKE_CASE__       (							a__      ):
    lowercase__          =      VOCAB_FILES_NAMES
    lowercase__          =      PRETRAINED_VOCAB_FILES_MAP
    lowercase__          =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase__          =      ["input_ids", "attention_mask"]
    lowercase__          =      RobertaTokenizer

    def __init__( self	: Optional[Any]       ,     lowerCAmelCase_	: Any=None       ,     lowerCAmelCase_	: str=None       ,     lowerCAmelCase_	: List[str]=None       ,     lowerCAmelCase_	: Tuple="replace"       ,     lowerCAmelCase_	: Any="<s>"       ,     lowerCAmelCase_	: List[Any]="</s>"       ,     lowerCAmelCase_	: Union[str, Any]="</s>"       ,     lowerCAmelCase_	: Any="<s>"       ,     lowerCAmelCase_	: Optional[Any]="<unk>"       ,     lowerCAmelCase_	: int="<pad>"       ,     lowerCAmelCase_	: Optional[Any]="<mask>"       ,     lowerCAmelCase_	: Any=False       ,     lowerCAmelCase_	: Dict=True       ,     **lowerCAmelCase_	: str       ,     ):







      """simple docstring"""







      super().__init__(
          __lowerCAmelCase       ,     __lowerCAmelCase       ,     tokenizer_file=__lowerCAmelCase       ,     errors=__lowerCAmelCase       ,     bos_token=__lowerCAmelCase       ,     eos_token=__lowerCAmelCase       ,     sep_token=__lowerCAmelCase       ,     cls_token=__lowerCAmelCase       ,     unk_token=__lowerCAmelCase       ,     pad_token=__lowerCAmelCase       ,     mask_token=__lowerCAmelCase       ,     add_prefix_space=__lowerCAmelCase       ,     trim_offsets=__lowerCAmelCase       ,     **__lowerCAmelCase       ,     )

      lowercase_	    =       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
      if pre_tok_state.get("""add_prefix_space"""       ,     __lowerCAmelCase) != add_prefix_space:
        lowercase_	    =       getattr(__lowerCAmelCase       ,     pre_tok_state.pop("""type"""))
        lowercase_	    =       add_prefix_space
        lowercase_	    =       pre_tok_class(**__lowerCAmelCase)

      lowercase_	    =       add_prefix_space

      lowercase_	    =       '''post_processor'''
      lowercase_	    =       getattr(self.backend_tokenizer       ,     __lowerCAmelCase       ,     __lowerCAmelCase)
      if tokenizer_component_instance:
        lowercase_	    =       json.loads(tokenizer_component_instance.__getstate__())

        # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
        if "sep" in state:
          lowercase_	    =       tuple(state["""sep"""])
        if "cls" in state:
          lowercase_	    =       tuple(state["""cls"""])

        lowercase_	    =       False

        if state.get("""add_prefix_space"""       ,     __lowerCAmelCase) != add_prefix_space:
          lowercase_	    =       add_prefix_space
          lowercase_	    =       True

        if state.get("""trim_offsets"""       ,     __lowerCAmelCase) != trim_offsets:
          lowercase_	    =       trim_offsets
          lowercase_	    =       True

        if changes_to_apply:
          lowercase_	    =       getattr(__lowerCAmelCase       ,     state.pop("""type"""))
          lowercase_	    =       component_class(**__lowerCAmelCase)
          setattr(self.backend_tokenizer       ,     __lowerCAmelCase       ,     __lowerCAmelCase)

    @property
    def     _UpperCAmelCase		( self	: str):







      """simple docstring"""







      if self._mask_token is None:
        if self.verbose:
          logger.error("""Using mask_token, but it is not set yet.""")
        return None
      return str(self._mask_token)

    @mask_token.setter
    def     _UpperCAmelCase		( self	: Optional[int]       ,     lowerCAmelCase_	: List[Any]):







      """simple docstring"""







      lowercase_	    =       AddedToken(__lowerCAmelCase       ,     lstrip=__lowerCAmelCase       ,     rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase       ,     __lowerCAmelCase) else value
      lowercase_	    =       value

    def     _UpperCAmelCase		( self	: int       ,     *lowerCAmelCase_	: int       ,     **lowerCAmelCase_	: str):







      """simple docstring"""







      lowercase_	    =       kwargs.get("""is_split_into_words"""       ,     __lowerCAmelCase)
      assert self.add_prefix_space or not is_split_into_words, (
          F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
          "to use it with pretokenized inputs."
      )

      return super()._batch_encode_plus(*__lowerCAmelCase       ,     **__lowerCAmelCase)

    def     _UpperCAmelCase		( self	: int       ,     *lowerCAmelCase_	: Optional[Any]       ,     **lowerCAmelCase_	: Tuple):







      """simple docstring"""







      lowercase_	    =       kwargs.get("""is_split_into_words"""       ,     __lowerCAmelCase)

      assert self.add_prefix_space or not is_split_into_words, (
          F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
          "to use it with pretokenized inputs."
      )

      return super()._encode_plus(*__lowerCAmelCase       ,     **__lowerCAmelCase)

    def     _UpperCAmelCase		( self	: Union[str, Any]       ,     lowerCAmelCase_	: Optional[Any]       ,     lowerCAmelCase_	: int = None):







      """simple docstring"""







      lowercase_	    =       self._tokenizer.model.save(__lowerCAmelCase       ,     name=__lowerCAmelCase)
      return tuple(__lowerCAmelCase)

    def     _UpperCAmelCase		( self	: str       ,     lowerCAmelCase_	: Tuple       ,     lowerCAmelCase_	: Any=None):







      """simple docstring"""







      lowercase_	    =       [self.bos_token_id] + token_ids_a + [self.eos_token_id]
      if token_ids_a is None:
        return output

      return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




    def     _UpperCAmelCase		( self	: Dict       ,     lowerCAmelCase_	: str       ,     lowerCAmelCase_	: List[str] = None):







      """simple docstring"""







      lowercase_	    =       [self.sep_token_id]
      lowercase_	    =       [self.cls_token_id]

      if token_ids_a is None:
        return len(cls + token_ids_a + sep) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]


'''simple docstring'''

from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import convert_to_rgb, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
  import PIL


__a: List[str]          =   logging.get_logger(__name__)



class 			UpperCAmelCase  (						a__					):





       '''simple docstring'''



       SCREAMING_SNAKE_CASE							=    ["pixel_values"]

       def __init__(						self	,   __lowerCAmelCase = True	,   __lowerCAmelCase = None	,   __lowerCAmelCase = PILImageResampling.BICUBIC	,   __lowerCAmelCase = True	,   __lowerCAmelCase = 1 / 255	,   __lowerCAmelCase = True	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = True	,   **__lowerCAmelCase	,   )  ->  None:
        super().__init__(**__lowerCAmelCase )
        lowercase__				:    Optional[int]  =					size if size is not None else {'''height''': 384, '''width''': 384}
        lowercase__				:    Optional[Any]  =					get_size_dict(__lowerCAmelCase	,   default_to_square=__lowerCAmelCase )

        lowercase__				:    Dict  =					do_resize
        lowercase__				:    int  =					size
        lowercase__				:    int  =					resample
        lowercase__				:    Tuple  =					do_rescale
        lowercase__				:    int  =					rescale_factor
        lowercase__				:    int  =					do_normalize
        lowercase__				:    Optional[int]  =					image_mean if image_mean is not None else OPENAI_CLIP_MEAN
        lowercase__				:    Optional[int]  =					image_std if image_std is not None else OPENAI_CLIP_STD
        lowercase__				:    Tuple  =					do_convert_rgb

       def _lowerCAmelCase(						self	,   __lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase = PILImageResampling.BICUBIC	,   __lowerCAmelCase = None	,   **__lowerCAmelCase	,   )  ->  np.ndarray:
        lowercase__				:    Union[str, Any]  =					get_size_dict(__lowerCAmelCase	,   default_to_square=__lowerCAmelCase )
        if "height" not in size or "width" not in size:
         raise ValueError(F"""The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}""" )
        lowercase__				:    Any  =					(size['''height'''], size['''width'''])
        return resize(__lowerCAmelCase	,   size=__lowerCAmelCase	,   resample=__lowerCAmelCase	,   data_format=__lowerCAmelCase	,   **__lowerCAmelCase )

       def _lowerCAmelCase(						self	,   __lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase = None	,   **__lowerCAmelCase	,   )  ->  str:
        return rescale(__lowerCAmelCase	,   scale=__lowerCAmelCase	,   data_format=__lowerCAmelCase	,   **__lowerCAmelCase )

       def _lowerCAmelCase(						self	,   __lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase = None	,   **__lowerCAmelCase	,   )  ->  np.ndarray:
        return normalize(__lowerCAmelCase	,   mean=__lowerCAmelCase	,   std=__lowerCAmelCase	,   data_format=__lowerCAmelCase	,   **__lowerCAmelCase )


       def _lowerCAmelCase(						self	,   __lowerCAmelCase	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = None	,   __lowerCAmelCase = ChannelDimension.FIRST	,   **__lowerCAmelCase	,   )  ->  PIL.Image.Image:
        lowercase__				:    Union[str, Any]  =					do_resize if do_resize is not None else self.do_resize
        lowercase__				:    Any  =					resample if resample is not None else self.resample
        lowercase__				:    int  =					do_rescale if do_rescale is not None else self.do_rescale
        lowercase__				:    List[Any]  =					rescale_factor if rescale_factor is not None else self.rescale_factor
        lowercase__				:    List[Any]  =					do_normalize if do_normalize is not None else self.do_normalize
        lowercase__				:    Dict  =					image_mean if image_mean is not None else self.image_mean
        lowercase__				:    Dict  =					image_std if image_std is not None else self.image_std
        lowercase__				:    Dict  =					do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

        lowercase__				:    Optional[int]  =					size if size is not None else self.size
        lowercase__				:    int  =					get_size_dict(__lowerCAmelCase	,   default_to_square=__lowerCAmelCase )

        lowercase__				:    str  =					make_list_of_images(__lowerCAmelCase )

        if not valid_images(__lowerCAmelCase ):
         raise ValueError(
             '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
             '''torch.Tensor, tf.Tensor or jax.ndarray.''' )

        if do_resize and size is None or resample is None:
         raise ValueError('''Size and resample must be specified if do_resize is True.''' )

        if do_rescale and rescale_factor is None:
         raise ValueError('''Rescale factor must be specified if do_rescale is True.''' )

        if do_normalize and (image_mean is None or image_std is None):
         raise ValueError('''Image mean and std must be specified if do_normalize is True.''' )

        # PIL RGBA images are converted to RGB
        if do_convert_rgb:
         lowercase__				:    Optional[Any]  =					[convert_to_rgb(__lowerCAmelCase ) for image in images]

        # All transformations expect numpy arrays.
        lowercase__				:    Any  =					[to_numpy_array(__lowerCAmelCase ) for image in images]

        if do_resize:
         lowercase__				:    Tuple  =					[self.resize(image=__lowerCAmelCase	,   size=__lowerCAmelCase	,   resample=__lowerCAmelCase ) for image in images]

        if do_rescale:
         lowercase__				:    List[str]  =					[self.rescale(image=__lowerCAmelCase	,   scale=__lowerCAmelCase ) for image in images]

        if do_normalize:
         lowercase__				:    Tuple  =					[self.normalize(image=__lowerCAmelCase	,   mean=__lowerCAmelCase	,   std=__lowerCAmelCase ) for image in images]

        lowercase__				:    List[str]  =					[to_channel_dimension_format(__lowerCAmelCase	,   __lowerCAmelCase ) for image in images]

        lowercase__				:    Optional[Any]  =					BatchFeature(data={'''pixel_values''': images}	,   tensor_type=__lowerCAmelCase )

        return encoded_outputs

