



'''simple docstring'''






from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
						import PIL

UpperCamelCase__:   Optional[Any]								=			logging.get_logger(__name__)


def  snake_case_     (					_lowerCAmelCase  :						List[Any]  )      ->   List[List[ImageInput]]:
							if isinstance(_lowerCAmelCase   ,      (list, tuple)  ) and isinstance(videos[0]   ,      (list, tuple)  ) and is_valid_image(videos[0][0]  ):
														return videos

							elif isinstance(_lowerCAmelCase   ,      (list, tuple)  ) and is_valid_image(videos[0]  ):
														return [videos]

							elif is_valid_image(_lowerCAmelCase  ):
														return [[videos]]

							raise ValueError(f"""Could not make batched video from {videos}"""  )




class 				SCREAMING_SNAKE_CASE(							A__			):







	"""simple docstring"""






	lowerCamelCase__			 =       ["""pixel_values"""]







	def __init__(						self			:     Optional[int]		,			__snake_case			:     bool = True		,			__snake_case			:     Dict[str, int] = None		,			__snake_case			:     PILImageResampling = PILImageResampling.BILINEAR		,			__snake_case			:     bool = True		,			__snake_case			:     Dict[str, int] = None		,			__snake_case			:     bool = True		,			__snake_case			:     Union[int, float] = 1 / 255		,			__snake_case			:     bool = True		,			__snake_case			:     bool = True		,			__snake_case			:     Optional[Union[float, List[float]]] = None		,			__snake_case			:     Optional[Union[float, List[float]]] = None		,			**__snake_case			:     Dict		,			)			->		None:
								super().__init__(**__snake_case		)
								UpperCAmelCase :       List[Any] 							=		size if size is not None else {'''shortest_edge''': 256}
								UpperCAmelCase :       Union[str, Any] 							=		get_size_dict(__snake_case		,			default_to_square=__snake_case		)
								UpperCAmelCase :       List[Any] 							=		crop_size if crop_size is not None else {'''height''': 224, '''width''': 224}
								UpperCAmelCase :       Union[str, Any] 							=		get_size_dict(__snake_case		,			param_name='''crop_size'''		)

								UpperCAmelCase :       Any 							=		do_resize
								UpperCAmelCase :       Optional[int] 							=		size
								UpperCAmelCase :       List[str] 							=		do_center_crop
								UpperCAmelCase :       Union[str, Any] 							=		crop_size
								UpperCAmelCase :       Dict 							=		resample
								UpperCAmelCase :       str 							=		do_rescale
								UpperCAmelCase :       List[str] 							=		rescale_factor
								UpperCAmelCase :       List[str] 							=		offset
								UpperCAmelCase :       Tuple 							=		do_normalize
								UpperCAmelCase :       Tuple 							=		image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
								UpperCAmelCase :       Optional[int] 							=		image_std if image_std is not None else IMAGENET_STANDARD_STD







	def      A	(						self			:     str		,			__snake_case			:     np.ndarray		,			__snake_case			:     Dict[str, int]		,			__snake_case			:     PILImageResampling = PILImageResampling.BILINEAR		,			__snake_case			:     Optional[Union[str, ChannelDimension]] = None		,			**__snake_case			:     Union[str, Any]		,			)			->		np.ndarray:
								UpperCAmelCase :       int 							=		get_size_dict(__snake_case		,			default_to_square=__snake_case		)
								if "shortest_edge" in size:
															UpperCAmelCase :       str 							=		get_resize_output_image_size(__snake_case		,			size['''shortest_edge''']		,			default_to_square=__snake_case		)
								elif "height" in size and "width" in size:
															UpperCAmelCase :       Optional[int] 							=		(size['''height'''], size['''width'''])
								else:
															raise ValueError(F"""Size must have 'height' and 'width' or 'shortest_edge' as keys. Got {size.keys()}"""		)
								return resize(__snake_case		,			size=__snake_case		,			resample=__snake_case		,			data_format=__snake_case		,			**__snake_case		)







	def      A	(						self			:     Dict		,			__snake_case			:     np.ndarray		,			__snake_case			:     Dict[str, int]		,			__snake_case			:     Optional[Union[str, ChannelDimension]] = None		,			**__snake_case			:     List[str]		,			)			->		np.ndarray:
								UpperCAmelCase :       Optional[Any] 							=		get_size_dict(__snake_case		)
								if "height" not in size or "width" not in size:
															raise ValueError(F"""Size must have 'height' and 'width' as keys. Got {size.keys()}"""		)
								return center_crop(__snake_case		,			size=(size['''height'''], size['''width'''])		,			data_format=__snake_case		,			**__snake_case		)







	def      A	(						self			:     Optional[int]		,			__snake_case			:     np.ndarray		,			__snake_case			:     Union[int, float]		,			__snake_case			:     bool = True		,			__snake_case			:     Optional[Union[str, ChannelDimension]] = None		,			**__snake_case			:     Optional[int]		,			)			->		Union[str, Any]:
								UpperCAmelCase :       List[str] 							=		image.astype(np.floataa		)
								if offset:
															UpperCAmelCase :       List[Any] 							=		image - (scale / 2)
								return rescale(__snake_case		,			scale=__snake_case		,			data_format=__snake_case		,			**__snake_case		)







	def      A	(						self			:     Optional[int]		,			__snake_case			:     np.ndarray		,			__snake_case			:     Union[float, List[float]]		,			__snake_case			:     Union[float, List[float]]		,			__snake_case			:     Optional[Union[str, ChannelDimension]] = None		,			**__snake_case			:     Optional[int]		,			)			->		np.ndarray:
								return normalize(__snake_case		,			mean=__snake_case		,			std=__snake_case		,			data_format=__snake_case		,			**__snake_case		)







	def      A	(						self			:     int		,			__snake_case			:     ImageInput		,			__snake_case			:     bool = None		,			__snake_case			:     Dict[str, int] = None		,			__snake_case			:     PILImageResampling = None		,			__snake_case			:     bool = None		,			__snake_case			:     Dict[str, int] = None		,			__snake_case			:     bool = None		,			__snake_case			:     float = None		,			__snake_case			:     bool = None		,			__snake_case			:     bool = None		,			__snake_case			:     Optional[Union[float, List[float]]] = None		,			__snake_case			:     Optional[Union[float, List[float]]] = None		,			__snake_case			:     Optional[ChannelDimension] = ChannelDimension.FIRST		,			)			->		np.ndarray:
								if do_resize and size is None or resample is None:
															raise ValueError('''Size and resample must be specified if do_resize is True.'''		)

								if do_center_crop and crop_size is None:
															raise ValueError('''Crop size must be specified if do_center_crop is True.'''		)

								if do_rescale and rescale_factor is None:
															raise ValueError('''Rescale factor must be specified if do_rescale is True.'''		)

								if do_normalize and (image_mean is None or image_std is None):
															raise ValueError('''Image mean and std must be specified if do_normalize is True.'''		)

								if offset and not do_rescale:
															raise ValueError('''For offset, do_rescale must also be set to True.'''		)

								# All transformations expect numpy arrays.
								UpperCAmelCase :       Optional[int] 							=		to_numpy_array(__snake_case		)

								if do_resize:
															UpperCAmelCase :       Optional[int] 							=		self.resize(image=__snake_case		,			size=__snake_case		,			resample=__snake_case		)

								if do_center_crop:
															UpperCAmelCase :       str 							=		self.center_crop(__snake_case		,			size=__snake_case		)

								if do_rescale:
															UpperCAmelCase :       Union[str, Any] 							=		self.rescale(image=__snake_case		,			scale=__snake_case		,			offset=__snake_case		)

								if do_normalize:
															UpperCAmelCase :       Tuple 							=		self.normalize(image=__snake_case		,			mean=__snake_case		,			std=__snake_case		)

								UpperCAmelCase :       int 							=		to_channel_dimension_format(__snake_case		,			__snake_case		)
								return image







	def      A	(						self			:     Optional[Any]		,			__snake_case			:     ImageInput		,			__snake_case			:     bool = None		,			__snake_case			:     Dict[str, int] = None		,			__snake_case			:     PILImageResampling = None		,			__snake_case			:     bool = None		,			__snake_case			:     Dict[str, int] = None		,			__snake_case			:     bool = None		,			__snake_case			:     float = None		,			__snake_case			:     bool = None		,			__snake_case			:     bool = None		,			__snake_case			:     Optional[Union[float, List[float]]] = None		,			__snake_case			:     Optional[Union[float, List[float]]] = None		,			__snake_case			:     Optional[Union[str, TensorType]] = None		,			__snake_case			:     ChannelDimension = ChannelDimension.FIRST		,			**__snake_case			:     int		,			)			->		PIL.Image.Image:
								UpperCAmelCase :       Optional[int] 							=		do_resize if do_resize is not None else self.do_resize
								UpperCAmelCase :       Tuple 							=		resample if resample is not None else self.resample
								UpperCAmelCase :       List[str] 							=		do_center_crop if do_center_crop is not None else self.do_center_crop
								UpperCAmelCase :       Optional[Any] 							=		do_rescale if do_rescale is not None else self.do_rescale
								UpperCAmelCase :       List[str] 							=		rescale_factor if rescale_factor is not None else self.rescale_factor
								UpperCAmelCase :       Dict 							=		offset if offset is not None else self.offset
								UpperCAmelCase :       int 							=		do_normalize if do_normalize is not None else self.do_normalize
								UpperCAmelCase :       Any 							=		image_mean if image_mean is not None else self.image_mean
								UpperCAmelCase :       str 							=		image_std if image_std is not None else self.image_std

								UpperCAmelCase :       Optional[int] 							=		size if size is not None else self.size
								UpperCAmelCase :       Optional[Any] 							=		get_size_dict(__snake_case		,			default_to_square=__snake_case		)
								UpperCAmelCase :       str 							=		crop_size if crop_size is not None else self.crop_size
								UpperCAmelCase :       str 							=		get_size_dict(__snake_case		,			param_name='''crop_size'''		)

								if not valid_images(__snake_case		):
															raise ValueError(
															    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
															    '''torch.Tensor, tf.Tensor or jax.ndarray.'''		)

								UpperCAmelCase :       Union[str, Any] 							=		make_batched(__snake_case		)

								UpperCAmelCase :       List[Any] 							=		[
								    [
								        self._preprocess_image(
								            image=__snake_case		,			do_resize=__snake_case		,			size=__snake_case		,			resample=__snake_case		,			do_center_crop=__snake_case		,			crop_size=__snake_case		,			do_rescale=__snake_case		,			rescale_factor=__snake_case		,			offset=__snake_case		,			do_normalize=__snake_case		,			image_mean=__snake_case		,			image_std=__snake_case		,			data_format=__snake_case		,			)
								        for img in video
								    ]
								    for video in videos
								]

								UpperCAmelCase :       Dict 							=		{'''pixel_values''': videos}
								return BatchFeature(data=__snake_case		,			tensor_type=__snake_case		)










'''simple docstring'''





import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowercase_		       =  logging.get_logger(__name__)

lowercase_		       =  "▁"

lowercase_		       =  {
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
    "tokenizer_config_file": "tokenizer_config.json",
}

lowercase_		       =  {
    "vocab_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json",
    },
    "spm_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_config_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json",
    },
}

lowercase_		       =  {
    "facebook/m2m100_418M": 1_024,
}

# fmt: off
lowercase_		       =  {
    "m2m100": ["af", "am", "ar", "ast", "az", "ba", "be", "bg", "bn", "br", "bs", "ca", "ceb", "cs", "cy", "da", "de", "el", "en", "es", "et", "fa", "ff", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "he", "hi", "hr", "ht", "hu", "hy", "id", "ig", "ilo", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "lb", "lg", "ln", "lo", "lt", "lv", "mg", "mk", "ml", "mn", "mr", "ms", "my", "ne", "nl", "no", "ns", "oc", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "so", "sq", "sr", "ss", "su", "sv", "sw", "ta", "th", "tl", "tn", "tr", "uk", "ur", "uz", "vi", "wo", "xh", "yi", "yo", "zh", "zu"],
    "wmt21": ["en", "ha", "is", "ja", "cs", "ru", "zh", "de"]
}






class __A			(  A						):



 '''simple docstring'''







 __lowerCamelCase    :       Union[str, Any]          =       VOCAB_FILES_NAMES
 __lowerCamelCase    :       Tuple          =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowerCamelCase    :       Tuple          =       PRETRAINED_VOCAB_FILES_MAP
 __lowerCamelCase    :       Dict          =       ['input_ids', 'attention_mask']

 __lowerCamelCase    :       List[int]          =       []
 __lowerCamelCase    :       List[int]          =       []



 def __init__(self ,   A ,   A ,   A=None ,   A=None ,   A="<s>" ,   A="</s>" ,   A="</s>" ,   A="<pad>" ,   A="<unk>" ,   A="m2m100" ,   A = None ,   A=8 ,   **A ,   )    ->     None:


      """simple docstring"""



      _a      =  {} if sp_model_kwargs is None else sp_model_kwargs

      _a      =  language_codes
      _a      =  FAIRSEQ_LANGUAGE_CODES[language_codes]
      _a      =  {lang_code: f'''__{lang_code}__''' for lang_code in fairseq_language_code}

      _a      =  kwargs.get('''additional_special_tokens''' ,   []       )
      kwargs["additional_special_tokens"] += [
          self.get_lang_token(A       )
          for lang_code in fairseq_language_code
          if self.get_lang_token(A       ) not in kwargs["additional_special_tokens"]
      ]

      super().__init__(
          src_lang=A ,   tgt_lang=A ,   bos_token=A ,   eos_token=A ,   sep_token=A ,   unk_token=A ,   pad_token=A ,   language_codes=A ,   sp_model_kwargs=self.sp_model_kwargs ,   num_madeup_words=A ,   **A ,   )

      _a      =  vocab_file
      _a      =  load_json(A       )
      _a      =  {v: k for k, v in self.encoder.items()}
      _a      =  spm_file
      _a      =  load_spm(A ,   self.sp_model_kwargs       )

      _a      =  len(self.encoder       )

      _a      =  {
          self.get_lang_token(A       ): self.encoder_size + i for i, lang_code in enumerate(A       )
      }
      _a      =  {lang_code: self.encoder_size + i for i, lang_code in enumerate(A       )}
      _a      =  {v: k for k, v in self.lang_token_to_id.items()}

      _a      =  src_lang if src_lang is not None else '''en'''
      _a      =  tgt_lang
      _a      =  self.get_lang_id(self._src_lang       )
      self.set_src_lang_special_tokens(self._src_lang       )

      _a      =  num_madeup_words



 @property
 def 					a__  (self       )    ->     int:


      """simple docstring"""



      return len(self.encoder       ) + len(self.lang_token_to_id       )



 @property
 def 					a__  (self       )    ->     str:


      """simple docstring"""



      return self._src_lang



 @src_lang.setter
 def 					a__  (self ,   A       )    ->     None:


      """simple docstring"""



      _a      =  new_src_lang
      self.set_src_lang_special_tokens(self._src_lang       )



 def 					a__  (self ,   A       )    ->     List[str]:


      """simple docstring"""



      return self.sp_model.encode(A ,   out_type=A       )



 def 					a__  (self ,   A       )    ->     Union[str, Any]:


      """simple docstring"""



      if token in self.lang_token_to_id:
           return self.lang_token_to_id[token]
      return self.encoder.get(A ,   self.encoder[self.unk_token]       )



 def 					a__  (self ,   A       )    ->     str:


      """simple docstring"""



      if index in self.id_to_lang_token:
           return self.id_to_lang_token[index]
      return self.decoder.get(A ,   self.unk_token       )



 def 					a__  (self ,   A       )    ->     Dict:


      """simple docstring"""



      _a      =  []
      _a      =  ''''''
      for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
                out_string += self.sp_model.decode(A       ) + token
                _a      =  []
           else:
                current_sub_tokens.append(A       )
      out_string += self.sp_model.decode(A       )
      return out_string.strip()



 def 					a__  (self ,   A ,   A = None ,   A = False       )    ->     List[int]:


      """simple docstring"""



      if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=A ,   token_ids_a=A ,   already_has_special_tokens=A       )

      _a      =  [1] * len(self.prefix_tokens       )
      _a      =  [1] * len(self.suffix_tokens       )
      if token_ids_a is None:
           return prefix_ones + ([0] * len(A       )) + suffix_ones
      return prefix_ones + ([0] * len(A       )) + ([0] * len(A       )) + suffix_ones



 def 					a__  (self ,   A ,   A = None       )    ->     List[int]:


      """simple docstring"""



      if token_ids_a is None:
           return self.prefix_tokens + token_ids_a + self.suffix_tokens
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



 def 					a__  (self       )    ->     Dict:


      """simple docstring"""



      _a      =  {self.convert_ids_to_tokens(A       ): i for i in range(self.vocab_size       )}
      vocab.update(self.added_tokens_encoder       )
      return vocab



 def __getstate__(self       )    ->     Dict:


      """simple docstring"""



      _a      =  self.__dict__.copy()
      _a      =  None
      return state



 def __setstate__(self ,   A       )    ->     None:


      """simple docstring"""



      _a      =  d

      # for backward compatibility
      if not hasattr(self ,   '''sp_model_kwargs'''       ):
           _a      =  {}

      _a      =  load_spm(self.spm_file ,   self.sp_model_kwargs       )



 def 					a__  (self ,   A ,   A = None       )    ->     Tuple[str]:


      """simple docstring"""



      _a      =  Path(A       )
      if not save_dir.is_dir():
           raise OSError(f'''{save_directory} should be a directory'''       )
      _a      =  save_dir / (
          (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
      )
      _a      =  save_dir / (
          (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
      )

      save_json(self.encoder ,   A       )

      if os.path.abspath(self.spm_file       ) != os.path.abspath(A       ) and os.path.isfile(self.spm_file       ):
           copyfile(self.spm_file ,   A       )
      elif not os.path.isfile(self.spm_file       ):
           with open(A ,   '''wb'''       ) as fi:
                _a      =  self.sp_model.serialized_model_proto()
                fi.write(A       )

      return (str(A       ), str(A       ))



 def 					a__  (self ,   A ,   A = "en" ,   A = None ,   A = "ro" ,   **A ,   )    ->     BatchEncoding:


      """simple docstring"""



      _a      =  src_lang
      _a      =  tgt_lang
      self.set_src_lang_special_tokens(self.src_lang       )
      return super().prepare_seqaseq_batch(A ,   A ,   **A       )



 def 					a__  (self ,   A ,   A ,   A ,   **A       )    ->     Union[str, Any]:


      """simple docstring"""



      if src_lang is None or tgt_lang is None:
           raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''       )
      _a      =  src_lang
      _a      =  self(A ,   add_special_tokens=A ,   **A       )
      _a      =  self.get_lang_id(A       )
      _a      =  tgt_lang_id
      return inputs



 def 					a__  (self       )    ->     Optional[Any]:


      """simple docstring"""



      self.set_src_lang_special_tokens(self.src_lang       )



 def 					a__  (self       )    ->     Tuple:


      """simple docstring"""



      self.set_tgt_lang_special_tokens(self.tgt_lang       )



 def 					a__  (self ,   A       )    ->     None:


      """simple docstring"""



      _a      =  self.get_lang_token(A       )
      _a      =  self.lang_token_to_id[lang_token]
      _a      =  [self.cur_lang_id]
      _a      =  [self.eos_token_id]



 def 					a__  (self ,   A       )    ->     None:


      """simple docstring"""



      _a      =  self.get_lang_token(A       )
      _a      =  self.lang_token_to_id[lang_token]
      _a      =  [self.cur_lang_id]
      _a      =  [self.eos_token_id]



 def 					a__  (self ,   A       )    ->     str:


      """simple docstring"""



      return self.lang_code_to_token[lang]



 def 					a__  (self ,   A       )    ->     int:


      """simple docstring"""



      _a      =  self.get_lang_token(A       )
      return self.lang_token_to_id[lang_token]





def lowerCAmelCase  (__A  ,	__A):



     """simple docstring"""



     _a      =  sentencepiece.SentencePieceProcessor(**__A)
     spm.Load(str(__A))
     return spm




def lowerCAmelCase  (__A):



     """simple docstring"""



     with open(__A  ,	'''r''') as f:
          return json.load(__A)




def lowerCAmelCase  (__A  ,	__A):



     """simple docstring"""



     with open(__A  ,	'''w''') as f:
          json.dump(__A  ,	__A  ,	indent=2)



