

import os
import sys
import warnings
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen
from ..table import array_cast
from ..utils.file_utils import is_local_path
from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
					import PIL.Image

					from .features import FeatureType


_UpperCAmelCase		:   Optional[List[str]]										=       None
_UpperCAmelCase		:   str										=       "<" if sys.byteorder == "little" else ">"
# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
_UpperCAmelCase		:   str										=       [
    np.dtype("""|b1"""),
    np.dtype("""|u1"""),
    np.dtype("""<u2"""),
    np.dtype(""">u2"""),
    np.dtype("""<i2"""),
    np.dtype(""">i2"""),
    np.dtype("""<u4"""),
    np.dtype(""">u4"""),
    np.dtype("""<i4"""),
    np.dtype(""">i4"""),
    np.dtype("""<f4"""),
    np.dtype(""">f4"""),
    np.dtype("""<f8"""),
    np.dtype(""">f8"""),
]







@dataclass
class  lowercase				:
				__SCREAMING_SNAKE_CASE					: str       			=			True
				__SCREAMING_SNAKE_CASE					: Union[str, Any]       			=			None
				# Automatically constructed
				__SCREAMING_SNAKE_CASE					: Tuple       			=			'''PIL.Image.Image'''
				__SCREAMING_SNAKE_CASE					: int       			=			pa.struct({'''bytes''': pa.binary(), '''path''': pa.string()}			)
				__SCREAMING_SNAKE_CASE					: int       			=			field(default='''Image'''	,  init=lowercase__	,  repr=lowercase__			)




				def __call__(					self       ):
											return self.pa_type




				def     a							(					self      ,   snake_case       ):



											if config.PIL_AVAILABLE:
																		import PIL.Image
											else:
																		raise ImportError('To support encoding images, please install \'Pillow\'.'       )

											if isinstance(__lowerCamelCase      ,   __lowerCamelCase       ):
																		snake_case_							=    np.array(__lowerCamelCase       )

											if isinstance(__lowerCamelCase      ,   __lowerCamelCase       ):
																		return {"path": value, "bytes": None}
											elif isinstance(__lowerCamelCase      ,   __lowerCamelCase       ):
																		return {"path": None, "bytes": value}
											elif isinstance(__lowerCamelCase      ,   np.ndarray       ):
																		# convert the image array to PNG/TIFF bytes
																		return encode_np_array(__lowerCamelCase       )
											elif isinstance(__lowerCamelCase      ,   PIL.Image.Image       ):
																		# convert the PIL image to bytes (default format is PNG/TIFF)
																		return encode_pil_image(__lowerCamelCase       )
											elif value.get('path'       ) is not None and os.path.isfile(value['path']       ):
																		# we set "bytes": None to not duplicate the data if they're already available locally
																		return {"bytes": None, "path": value.get('path'       )}
											elif value.get('bytes'       ) is not None or value.get('path'       ) is not None:
																		# store the image bytes, and path is used to infer the image format using the file extension
																		return {"bytes": value.get('bytes'       ), "path": value.get('path'       )}
											else:
																		raise ValueError(
																		    F'''An image sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.'''       )




				def     a							(					self      ,   snake_case      ,   snake_case=None       ):



											if not self.decode:
																		raise RuntimeError('Decoding is disabled for this feature. Please use Image(decode=True) instead.'       )

											if config.PIL_AVAILABLE:
																		import PIL.Image
											else:
																		raise ImportError('To support decoding images, please install \'Pillow\'.'       )

											if token_per_repo_id is None:
																		snake_case_							=    {}

											snake_case_							=    value["path"], value["bytes"]
											if bytes_ is None:
																		if path is None:
																									raise ValueError(F'''An image should have one of \'path\' or \'bytes\' but both are None in {value}.'''       )
																		else:
																									if is_local_path(__lowerCamelCase       ):
																																snake_case_							=    PIL.Image.open(__lowerCamelCase       )
																									else:
																																snake_case_							=    path.split('::'       )[-1]
																																try:
																																							snake_case_							=    string_to_dict(__lowerCamelCase      ,   config.HUB_DATASETS_URL       )["repo_id"]
																																							snake_case_							=    token_per_repo_id.get(__lowerCamelCase       )
																																except ValueError:
																																							snake_case_							=    None
																																with xopen(__lowerCamelCase      ,   'rb'      ,   use_auth_token=__lowerCamelCase       ) as f:
																																							snake_case_							=    BytesIO(f.read()       )
																																snake_case_							=    PIL.Image.open(bytes_       )
											else:
																		snake_case_							=    PIL.Image.open(BytesIO(bytes_       )       )
											image.load()  # to avoid "Too many open files" errors
											return image




				def     a							(					self       ):
											from .features import Value

											return (
											    self
											    if self.decode
											    else {
											        "bytes": Value('binary'       ),
											        "path": Value('string'       ),
											    }
											)




				def     a							(					self      ,   snake_case       ):



											if pa.types.is_string(storage.type       ):
																		snake_case_							=    pa.array([None] * len(__lowerCamelCase       )      ,   type=pa.binary()       )
																		snake_case_							=    pa.StructArray.from_arrays([bytes_array, storage]      ,   ['bytes', 'path']      ,   mask=storage.is_null()       )
											elif pa.types.is_binary(storage.type       ):
																		snake_case_							=    pa.array([None] * len(__lowerCamelCase       )      ,   type=pa.string()       )
																		snake_case_							=    pa.StructArray.from_arrays([storage, path_array]      ,   ['bytes', 'path']      ,   mask=storage.is_null()       )
											elif pa.types.is_struct(storage.type       ):
																		if storage.type.get_field_index('bytes'       ) >= 0:
																									snake_case_							=    storage.field('bytes'       )
																		else:
																									snake_case_							=    pa.array([None] * len(__lowerCamelCase       )      ,   type=pa.binary()       )
																		if storage.type.get_field_index('path'       ) >= 0:
																									snake_case_							=    storage.field('path'       )
																		else:
																									snake_case_							=    pa.array([None] * len(__lowerCamelCase       )      ,   type=pa.string()       )
																		snake_case_							=    pa.StructArray.from_arrays([bytes_array, path_array]      ,   ['bytes', 'path']      ,   mask=storage.is_null()       )
											elif pa.types.is_list(storage.type       ):
																		snake_case_							=    pa.array(
																		    [encode_np_array(np.array(__lowerCamelCase       )       )['bytes'] if arr is not None else None for arr in storage.to_pylist()]      ,   type=pa.binary()      ,   )
																		snake_case_							=    pa.array([None] * len(__lowerCamelCase       )      ,   type=pa.string()       )
																		snake_case_							=    pa.StructArray.from_arrays(
																		    [bytes_array, path_array]      ,   ['bytes', 'path']      ,   mask=bytes_array.is_null()       )
											return array_cast(__lowerCamelCase      ,   self.pa_type       )




				def     a							(					self      ,   snake_case       ):



											@no_op_if_value_is_null
											def path_to_bytes(snake_case       ):
																		with xopen(__lowerCamelCase      ,   'rb'       ) as f:
																									snake_case_							=    f.read()
																		return bytes_

											snake_case_							=    pa.array(
											    [
											        (path_to_bytes(x['path']       ) if x['bytes'] is None else x['bytes']) if x is not None else None
											        for x in storage.to_pylist()
											    ]      ,   type=pa.binary()      ,   )
											snake_case_							=    pa.array(
											    [os.path.basename(__lowerCamelCase       ) if path is not None else None for path in storage.field('path'       ).to_pylist()]      ,   type=pa.string()      ,   )
											snake_case_							=    pa.StructArray.from_arrays([bytes_array, path_array]      ,   ['bytes', 'path']      ,   mask=bytes_array.is_null()       )
											return array_cast(__lowerCamelCase      ,   self.pa_type       )





def 					__lowerCamelCase	( ):







							'''simple docstring'''






							if config.PIL_AVAILABLE:
														import PIL.Image
							else:
														raise ImportError('To support encoding images, please install \'Pillow\'.'      )

							global _IMAGE_COMPRESSION_FORMATS
							if _IMAGE_COMPRESSION_FORMATS is None:
														PIL.Image.init()
														snake_case_							=    list(set(PIL.Image.OPEN.keys()      ) & set(PIL.Image.SAVE.keys()      )      )
							return _IMAGE_COMPRESSION_FORMATS





def 					__lowerCamelCase	( UpperCamelCase__      ):







							'''simple docstring'''






							snake_case_							=    BytesIO()
							if image.format in list_image_compression_formats():
														snake_case_							=    image.format
							else:
														snake_case_							=    "PNG" if image.mode in ["1", "L", "LA", "RGB", "RGBA"] else "TIFF"
							image.save(_A   ,							format=_A      )
							return buffer.getvalue()





def 					__lowerCamelCase	( UpperCamelCase__      ):







							'''simple docstring'''






							if hasattr(_A   ,							'filename'      ) and image.filename != "":
														return {"path": image.filename, "bytes": None}
							else:
														return {"path": None, "bytes": image_to_bytes(_A      )}





def 					__lowerCamelCase	( UpperCamelCase__      ):







							'''simple docstring'''






							if config.PIL_AVAILABLE:
														import PIL.Image
							else:
														raise ImportError('To support encoding images, please install \'Pillow\'.'      )

							snake_case_							=    array.dtype
							snake_case_							=    dtype.byteorder if dtype.byteorder != "=" else _NATIVE_BYTEORDER
							snake_case_							=    dtype.kind
							snake_case_							=    dtype.itemsize

							snake_case_							=    None

							# Multi-channel array case (only np.dtype("|u1") is allowed)
							if array.shape[2:]:
														snake_case_							=    np.dtype('|u1'      )
														if dtype_kind not in ["u", "i"]:
																					raise TypeError(
																					    F'''Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays.'''      )
														if dtype is not dest_dtype:
																					warnings.warn(F'''Downcasting array dtype {dtype} to {dest_dtype} to be compatible with \'Pillow\''''      )
    # Exact match
							elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
														snake_case_							=    dtype
							else:  # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
														while dtype_itemsize >= 1:
																					snake_case_							=    dtype_byteorder + dtype_kind + str(_A      )
																					snake_case_							=    np.dtype(_A      )
																					if dest_dtype in _VALID_IMAGE_ARRAY_DTPYES:
																												warnings.warn(F'''Downcasting array dtype {dtype} to {dest_dtype} to be compatible with \'Pillow\''''      )
																												break
																					else:
																												dtype_itemsize //= 2

							if dest_dtype is None:
														raise TypeError(
														    F'''Cannot convert dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}'''      )
							snake_case_							=    PIL.Image.fromarray(array.astype(_A      )      )
							return {"path": None, "bytes": image_to_bytes(_A      )}





def 					__lowerCamelCase	( UpperCamelCase__      ):







							'''simple docstring'''






							if config.PIL_AVAILABLE:
														import PIL.Image
							else:
														raise ImportError('To support encoding images, please install \'Pillow\'.'      )

							if objs:
														snake_case_							=    first_non_null_value(_A      )
														if isinstance(_A   ,							_A      ):
																					return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
														if isinstance(_A   ,							np.ndarray      ):
																					snake_case_							=    no_op_if_value_is_null(_A      )
																					return [obj_to_image_dict_func(_A      ) for obj in objs]
														elif isinstance(_A   ,							PIL.Image.Image      ):
																					snake_case_							=    no_op_if_value_is_null(_A      )
																					return [obj_to_image_dict_func(_A      ) for obj in objs]
														else:
																					return objs
							else:
														return objs








from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


A      :    Union[str, Any]		=					logging.get_logger(__name__)

A      :    Optional[Any]		=					{
    "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/config.json",
    "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/config.json",
    "xlm-roberta-large-finetuned-conll02-dutch": (
        "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/config.json"
    ),
    "xlm-roberta-large-finetuned-conll02-spanish": (
        "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/config.json"
    ),
    "xlm-roberta-large-finetuned-conll03-english": (
        "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/config.json"
    ),
    "xlm-roberta-large-finetuned-conll03-german": (
        "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/config.json"
    ),
}







class 			_lowercase (							lowercase__):


       """simple docstring"""




       A__											=			"xlm-roberta"





       def __init__(   self							:			Union[str, Any]					,      __lowerCamelCase							:			Optional[Any]=30522					,      __lowerCamelCase							:			List[Any]=768					,      __lowerCamelCase							:			Union[str, Any]=12					,      __lowerCamelCase							:			int=12					,      __lowerCamelCase							:			Dict=3072					,      __lowerCamelCase							:			Optional[int]="gelu"					,      __lowerCamelCase							:			Optional[Any]=0.1					,      __lowerCamelCase							:			Tuple=0.1					,      __lowerCamelCase							:			str=512					,      __lowerCamelCase							:			Optional[Any]=2					,      __lowerCamelCase							:			Dict=0.0_2					,      __lowerCamelCase							:			List[str]=1E-1_2					,      __lowerCamelCase							:			Optional[int]=1					,      __lowerCamelCase							:			Union[str, Any]=0					,      __lowerCamelCase							:			Optional[int]=2					,      __lowerCamelCase							:			Dict="absolute"					,      __lowerCamelCase							:			Optional[Any]=True					,      __lowerCamelCase							:			Tuple=None					,      **__lowerCamelCase							:			int					,      ):


             '''simple docstring'''



             super().__init__(pad_token_id=__lowerCamelCase					,      bos_token_id=__lowerCamelCase					,      eos_token_id=__lowerCamelCase					,      **__lowerCamelCase       )

             lowerCamelCase__		:				str    =  vocab_size
             lowerCamelCase__		:				Any    =  hidden_size
             lowerCamelCase__		:				Optional[Any]    =  num_hidden_layers
             lowerCamelCase__		:				Union[str, Any]    =  num_attention_heads
             lowerCamelCase__		:				Any    =  hidden_act
             lowerCamelCase__		:				str    =  intermediate_size
             lowerCamelCase__		:				Optional[int]    =  hidden_dropout_prob
             lowerCamelCase__		:				Any    =  attention_probs_dropout_prob
             lowerCamelCase__		:				Union[str, Any]    =  max_position_embeddings
             lowerCamelCase__		:				List[str]    =  type_vocab_size
             lowerCamelCase__		:				Tuple    =  initializer_range
             lowerCamelCase__		:				Tuple    =  layer_norm_eps
             lowerCamelCase__		:				Any    =  position_embedding_type
             lowerCamelCase__		:				Any    =  use_cache
             lowerCamelCase__		:				Any    =  classifier_dropout







class 			_lowercase (							lowercase__):


       """simple docstring"""




       @property
       def 						lowerCAmelCase     (   self							:			Union[str, Any]       ):


             '''simple docstring'''



             if self.task == "multiple-choice":
                   lowerCamelCase__		:				List[str]    =  {0: "batch", 1: "choice", 2: "sequence"}
             else:
                   lowerCamelCase__		:				Any    =  {0: "batch", 1: "sequence"}
             return OrderedDict(
                 [
                     ("input_ids", dynamic_axis),
                     ("attention_mask", dynamic_axis),
                 ]       )


