





"""simple docstring"""




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_xlm_roberta import XLMRobertaTokenizer
else:
   lowerCamelCase_	:		Dict         =   None


lowerCamelCase_	:		List[Any]         =   logging.get_logger(__name__)

lowerCamelCase_	:		List[str]         =   {'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase_	:		Dict         =   {
    'vocab_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/tokenizer.json',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/tokenizer.json'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/tokenizer.json'
        ),
    },
}

lowerCamelCase_	:		Dict         =   {
    'xlm-roberta-base': 5_12,
    'xlm-roberta-large': 5_12,
    'xlm-roberta-large-finetuned-conll02-dutch': 5_12,
    'xlm-roberta-large-finetuned-conll02-spanish': 5_12,
    'xlm-roberta-large-finetuned-conll03-english': 5_12,
    'xlm-roberta-large-finetuned-conll03-german': 5_12,
}

class 					_UpperCAmelCase    (  UpperCAmelCase__						):



      '''simple docstring'''
      lowercase_ :					Tuple             =  VOCAB_FILES_NAMES
      lowercase_ :					str             =  PRETRAINED_VOCAB_FILES_MAP
      lowercase_ :					Dict             =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowercase_ :					List[Any]             =  ["""input_ids""", """attention_mask"""]
      lowercase_ :					Union[str, Any]             =  XLMRobertaTokenizer

      def __init__(	self   , snake_case_=None   , snake_case_=None   , snake_case_="<s>"   , snake_case_="</s>"   , snake_case_="</s>"   , snake_case_="<s>"   , snake_case_="<unk>"   , snake_case_="<pad>"   , snake_case_="<mask>"   , **snake_case_   , ):

        """simple docstring"""

        A_    :			List[Any]									=    AddedToken(snake_case_   , lstrip=snake_case_   , rstrip=snake_case_    ) if isinstance(snake_case_   , snake_case_    ) else mask_token

        super().__init__(
            snake_case_   , tokenizer_file=snake_case_   , bos_token=snake_case_   , eos_token=snake_case_   , sep_token=snake_case_   , cls_token=snake_case_   , unk_token=snake_case_   , pad_token=snake_case_   , mask_token=snake_case_   , **snake_case_   , )

        A_    :			Any									=    vocab_file
        A_    :			str									=    False if not self.vocab_file else True

      def    lowerCamelCase_     (	self   , snake_case_   , snake_case_ = None    ):

        """simple docstring"""

        if token_ids_a is None:
          return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        A_    :			Any									=    [self.cls_token_id]
        A_    :			List[Any]									=    [self.sep_token_id]
        return cls + token_ids_a + sep + sep + token_ids_a + sep

      def    lowerCamelCase_     (	self   , snake_case_   , snake_case_ = None    ):

        """simple docstring"""

        A_    :			int									=    [self.sep_token_id]
        A_    :			str									=    [self.cls_token_id]

        if token_ids_a is None:
          return len(cls + token_ids_a + sep    ) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]

      def    lowerCamelCase_     (	self   , snake_case_   , snake_case_ = None    ):

        """simple docstring"""

        if not self.can_save_slow_tokenizer:
          raise ValueError(
              'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
              'tokenizer.'    )

        if not os.path.isdir(snake_case_    ):
          logger.error(F"""Vocabulary path ({save_directory}) should be a directory."""    )
          return
        A_    :			Tuple									=    os.path.join(
            snake_case_   , (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )

        if os.path.abspath(self.vocab_file    ) != os.path.abspath(snake_case_    ):
          copyfile(self.vocab_file   , snake_case_    )

        return (out_vocab_file,)





import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
				import torch

if is_vision_available():
				from PIL import Image

				from transformers import CLIPImageProcessor




class     A					(				unittest.TestCase       ):







							def __init__(    self   :	Any   ,     __UpperCAmelCase   :	Optional[int]   ,     __UpperCAmelCase   :	List[Any]=7   ,     __UpperCAmelCase   :	Union[str, Any]=3   ,     __UpperCAmelCase   :	Optional[Any]=18   ,     __UpperCAmelCase   :	List[str]=30   ,     __UpperCAmelCase   :	int=400   ,     __UpperCAmelCase   :	List[Any]=True   ,     __UpperCAmelCase   :	Tuple=None   ,     __UpperCAmelCase   :	Optional[Any]=True   ,     __UpperCAmelCase   :	Any=None   ,     __UpperCAmelCase   :	Optional[int]=True   ,     __UpperCAmelCase   :	List[str]=[0.48_145_466, 0.4_578_275, 0.40_821_073]   ,     __UpperCAmelCase   :	List[Any]=[0.26_862_954, 0.26_130_258, 0.27_577_711]   ,     __UpperCAmelCase   :	Dict=True   ,     )		->					Tuple:

										"""simple docstring"""



										UpperCamelCase_     				= size if size is not None else {'shortest_edge': 20}
										UpperCamelCase_     				= crop_size if crop_size is not None else {'height': 18, 'width': 18}
										UpperCamelCase_     				= parent
										UpperCamelCase_     				= batch_size
										UpperCamelCase_     				= num_channels
										UpperCamelCase_     				= image_size
										UpperCamelCase_     				= min_resolution
										UpperCamelCase_     				= max_resolution
										UpperCamelCase_     				= do_resize
										UpperCamelCase_     				= size
										UpperCamelCase_     				= do_center_crop
										UpperCamelCase_     				= crop_size
										UpperCamelCase_     				= do_normalize
										UpperCamelCase_     				= image_mean
										UpperCamelCase_     				= image_std
										UpperCamelCase_     				= do_convert_rgb







							def        lowercase__							(    self   :	Tuple				)		->					List[Any]:

										"""simple docstring"""



										return {
										    "do_resize": self.do_resize,
										    "size": self.size,
										    "do_center_crop": self.do_center_crop,
										    "crop_size": self.crop_size,
										    "do_normalize": self.do_normalize,
										    "image_mean": self.image_mean,
										    "image_std": self.image_std,
										    "do_convert_rgb": self.do_convert_rgb,
										}







							def        lowercase__							(    self   :	List[Any]   ,     __UpperCAmelCase   :	str=False   ,     __UpperCAmelCase   :	Tuple=False   ,     __UpperCAmelCase   :	Optional[int]=False				)		->					Any:

										"""simple docstring"""



										assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

										if equal_resolution:
													UpperCamelCase_     				= []
													for i in range(self.batch_size				):
																image_inputs.append(
																    np.random.randint(
																        255   ,     size=(self.num_channels, self.max_resolution, self.max_resolution)   ,     dtype=np.uinta				)				)
										else:
													UpperCamelCase_     				= []
													for i in range(self.batch_size				):
																UpperCamelCase_				,   UpperCamelCase_     				= np.random.choice(np.arange(self.min_resolution   ,     self.max_resolution				)   ,     2				)
																image_inputs.append(np.random.randint(255   ,     size=(self.num_channels, width, height)   ,     dtype=np.uinta				)				)

										if not numpify and not torchify:
													# PIL expects the channel dimension as last dimension
													UpperCamelCase_     				= [Image.fromarray(np.moveaxis(__UpperCAmelCase   ,     0   ,     -1				)				) for x in image_inputs]

										if torchify:
													UpperCamelCase_     				= [torch.from_numpy(__UpperCAmelCase				) for x in image_inputs]

										return image_inputs







@require_torch
@require_vision
class     A					(				lowerCamelCase_  ,						unittest.TestCase       ):
							_SCREAMING_SNAKE_CASE  :		Any			  =       CLIPImageProcessor if is_vision_available() else None







							def        lowercase__							(    self   :	Dict				)		->					List[str]:

										"""simple docstring"""



										UpperCamelCase_     				= CLIPImageProcessingTester(self				)







							@property
							def        lowercase__							(    self   :	Optional[int]				)		->					Optional[Any]:

										"""simple docstring"""



										return self.image_processor_tester.prepare_image_processor_dict()







							def        lowercase__							(    self   :	Tuple				)		->					Tuple:

										"""simple docstring"""



										UpperCamelCase_     				= self.image_processing_class(**self.image_processor_dict				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_resize'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'size'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_center_crop'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'center_crop'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_normalize'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'image_mean'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'image_std'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_convert_rgb'				)				)







							def        lowercase__							(    self   :	Optional[int]				)		->					Tuple:

										"""simple docstring"""



										UpperCamelCase_     				= self.image_processing_class.from_dict(self.image_processor_dict				)
										self.assertEqual(image_processor.size   ,     {'shortest_edge': 20}				)
										self.assertEqual(image_processor.crop_size   ,     {'height': 18, 'width': 18}				)

										UpperCamelCase_     				= self.image_processing_class.from_dict(self.image_processor_dict   ,     size=42   ,     crop_size=84				)
										self.assertEqual(image_processor.size   ,     {'shortest_edge': 42}				)
										self.assertEqual(image_processor.crop_size   ,     {'height': 84, 'width': 84}				)







							def        lowercase__							(    self   :	Union[str, Any]				)		->					Optional[int]:

										"""simple docstring"""



										pass







							def        lowercase__							(    self   :	int				)		->					Optional[int]:

										"""simple docstring"""



										UpperCamelCase_     				= self.image_processing_class(**self.image_processor_dict				)
										# create random PIL images
										UpperCamelCase_     				= self.image_processor_tester.prepare_inputs(equal_resolution=__UpperCAmelCase				)
										for image in image_inputs:
													self.assertIsInstance(__UpperCAmelCase   ,     Image.Image				)

										# Test not batched input
										UpperCamelCase_     				= image_processing(image_inputs[0]   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )

										# Test batched
										UpperCamelCase_     				= image_processing(__UpperCAmelCase   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )







							def        lowercase__							(    self   :	Optional[Any]				)		->					Tuple:

										"""simple docstring"""



										UpperCamelCase_     				= self.image_processing_class(**self.image_processor_dict				)
										# create random numpy tensors
										UpperCamelCase_     				= self.image_processor_tester.prepare_inputs(equal_resolution=__UpperCAmelCase   ,     numpify=__UpperCAmelCase				)
										for image in image_inputs:
													self.assertIsInstance(__UpperCAmelCase   ,     np.ndarray				)

										# Test not batched input
										UpperCamelCase_     				= image_processing(image_inputs[0]   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )

										# Test batched
										UpperCamelCase_     				= image_processing(__UpperCAmelCase   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )







							def        lowercase__							(    self   :	Optional[Any]				)		->					Any:

										"""simple docstring"""



										UpperCamelCase_     				= self.image_processing_class(**self.image_processor_dict				)
										# create random PyTorch tensors
										UpperCamelCase_     				= self.image_processor_tester.prepare_inputs(equal_resolution=__UpperCAmelCase   ,     torchify=__UpperCAmelCase				)
										for image in image_inputs:
													self.assertIsInstance(__UpperCAmelCase   ,     torch.Tensor				)

										# Test not batched input
										UpperCamelCase_     				= image_processing(image_inputs[0]   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )

										# Test batched
										UpperCamelCase_     				= image_processing(__UpperCAmelCase   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )







@require_torch
@require_vision
class     A					(				lowerCamelCase_  ,						unittest.TestCase       ):
							_SCREAMING_SNAKE_CASE  :		str			  =       CLIPImageProcessor if is_vision_available() else None







							def        lowercase__							(    self   :	str				)		->					Any:

										"""simple docstring"""



										UpperCamelCase_     				= CLIPImageProcessingTester(self   ,     num_channels=4				)
										UpperCamelCase_     				= 3







							@property
							def        lowercase__							(    self   :	Dict				)		->					str:

										"""simple docstring"""



										return self.image_processor_tester.prepare_image_processor_dict()







							def        lowercase__							(    self   :	Optional[int]				)		->					List[str]:

										"""simple docstring"""



										UpperCamelCase_     				= self.image_processing_class(**self.image_processor_dict				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_resize'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'size'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_center_crop'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'center_crop'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_normalize'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'image_mean'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'image_std'				)				)
										self.assertTrue(hasattr(__UpperCAmelCase   ,     'do_convert_rgb'				)				)







							def        lowercase__							(    self   :	int				)		->					Any:

										"""simple docstring"""



										pass







							def        lowercase__							(    self   :	Any				)		->					Optional[int]:

										"""simple docstring"""



										UpperCamelCase_     				= self.image_processing_class(**self.image_processor_dict				)
										# create random PIL images
										UpperCamelCase_     				= self.image_processor_tester.prepare_inputs(equal_resolution=__UpperCAmelCase				)
										for image in image_inputs:
													self.assertIsInstance(__UpperCAmelCase   ,     Image.Image				)

										# Test not batched input
										UpperCamelCase_     				= image_processing(image_inputs[0]   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        1,
										        self.expected_encoded_image_num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )

										# Test batched
										UpperCamelCase_     				= image_processing(__UpperCAmelCase   ,     return_tensors='pt'				).pixel_values
										self.assertEqual(
										    encoded_images.shape   ,     (
										        self.image_processor_tester.batch_size,
										        self.expected_encoded_image_num_channels,
										        self.image_processor_tester.crop_size['height'],
										        self.image_processor_tester.crop_size['width'],
										    )   ,     )
