




from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase__    :							Optional[int]       		=     logging.get_logger(__name__)

UpperCAmelCase__    :							int       		=     {
    'google/reformer-crime-and-punishment': (
        'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/config.json'
    ),
    'google/reformer-enwik8': 'https://huggingface.co/google/reformer-enwik8/resolve/main/config.json',
}

class   UpperCAmelCase    (      __SCREAMING_SNAKE_CASE ):
   __SCREAMING_SNAKE_CASE		:				str      					=       "reformer"
   __SCREAMING_SNAKE_CASE		:				Union[str, Any]      					=       ["past_buckets_states"]
   __SCREAMING_SNAKE_CASE		:				Union[str, Any]      					=       {}






   def __init__(						self :							List[Any]    ,       UpperCamelCase_ :							str=64    ,       UpperCamelCase_ :							List[str]=["local", "lsh", "local", "lsh", "local", "lsh"]    ,       UpperCamelCase_ :							Optional[int]=1.0    ,       UpperCamelCase_ :							List[Any]=True    ,       UpperCamelCase_ :							Union[str, Any]=[64, 64]    ,       UpperCamelCase_ :							str=[64, 1_92]    ,       UpperCamelCase_ :							str=0    ,       UpperCamelCase_ :							str=2    ,       UpperCamelCase_ :							int=5_12    ,       UpperCamelCase_ :							str=None    ,       UpperCamelCase_ :							Tuple="relu"    ,       UpperCamelCase_ :							Optional[int]=0.05    ,       UpperCamelCase_ :							int=2_56    ,       UpperCamelCase_ :							Tuple=0.02    ,       UpperCamelCase_ :							Dict=False    ,       UpperCamelCase_ :							Tuple=1E-12    ,       UpperCamelCase_ :							str=1    ,       UpperCamelCase_ :							Dict=0    ,       UpperCamelCase_ :							List[Any]=0.05    ,       UpperCamelCase_ :							int=64    ,       UpperCamelCase_ :							Any=64    ,       UpperCamelCase_ :							Any=0.0    ,       UpperCamelCase_ :							Dict=1    ,       UpperCamelCase_ :							int=0    ,       UpperCamelCase_ :							List[Any]=40_96    ,       UpperCamelCase_ :							Any=12    ,       UpperCamelCase_ :							str=None    ,       UpperCamelCase_ :							Any=1    ,       UpperCamelCase_ :							Dict=0    ,       UpperCamelCase_ :							Tuple=3_20    ,       UpperCamelCase_ :							Union[str, Any]=False    ,       UpperCamelCase_ :							Optional[Any]=True    ,       UpperCamelCase_ :							Optional[int]=None    ,       **UpperCamelCase_ :							Any    ,       )->      Tuple:
     _lowercase:			str      =     hash_seed
     _lowercase:			int      =     vocab_size
     _lowercase:			str      =     attention_head_size
     _lowercase:			List[Any]      =     hidden_size
     _lowercase:			Optional[int]      =     num_attention_heads
     _lowercase:			List[str]      =     num_hashes
     _lowercase:			Any      =     len(__UpperCAmelCase  )
     _lowercase:			List[str]      =     tuple(__UpperCAmelCase  ) if isinstance(__UpperCAmelCase    ,       __UpperCAmelCase  ) else num_buckets
     _lowercase:			str      =     lsh_attn_chunk_length
     _lowercase:			Optional[Any]      =     local_attn_chunk_length
     _lowercase:			str      =     lsh_num_chunks_after
     _lowercase:			Any      =     lsh_num_chunks_before
     _lowercase:			Any      =     local_num_chunks_after
     _lowercase:			Optional[int]      =     local_num_chunks_before
     _lowercase:			Union[str, Any]      =     hidden_act
     _lowercase:			Optional[int]      =     feed_forward_size
     _lowercase:			Union[str, Any]      =     hidden_dropout_prob
     _lowercase:			str      =     lsh_attention_probs_dropout_prob
     _lowercase:			Optional[int]      =     local_attention_probs_dropout_prob
     _lowercase:			List[str]      =     max_position_embeddings
     _lowercase:			Union[str, Any]      =     initializer_range
     _lowercase:			List[Any]      =     layer_norm_eps
     _lowercase:			Union[str, Any]      =     axial_pos_embds
     _lowercase:			List[str]      =     tuple(__UpperCAmelCase  )
     _lowercase:			str      =     tuple(__UpperCAmelCase  )
     _lowercase:			Optional[int]      =     axial_norm_std
     _lowercase:			Dict      =     chunk_size_lm_head
     _lowercase:			Optional[int]      =     attn_layers
     _lowercase:			Optional[Any]      =     use_cache
     _lowercase:			int      =     classifier_dropout
     super().__init__(
         pad_token_id=__UpperCAmelCase    ,       eos_token_id=__UpperCAmelCase    ,       is_decoder=__UpperCAmelCase    ,       tie_word_embeddings=__UpperCAmelCase    ,       **__UpperCAmelCase    ,       )





'''simple docstring'''
import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


_UpperCamelCase									=      logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
_UpperCamelCase									=      direct_transformers_import(Path(__file__).parent)


_UpperCamelCase									=      {
    '''AutoTokenizer''': '''PreTrainedTokenizerBase''',
    '''AutoFeatureExtractor''': '''FeatureExtractionMixin''',
    '''AutoImageProcessor''': '''ImageProcessingMixin''',
}



class _A	(     __SCREAMING_SNAKE_CASE     ):
		_SCREAMING_SNAKE_CASE       :     str       =      ["feature_extractor", "tokenizer"]
		# Names need to be attr_class for attr in attributes
		_SCREAMING_SNAKE_CASE       :     Optional[int]       =      None
		_SCREAMING_SNAKE_CASE       :     str       =      None
		_SCREAMING_SNAKE_CASE       :     List[str]       =      None





		def __init__(  self			,							*__UpperCAmelCase			,							**__UpperCAmelCase					)							->  Optional[int]:

					'''simple docstring'''
					# Sanitize args and kwargs
					for key in kwargs:
								if key not in self.attributes:
											raise TypeError(f'Unexpected keyword argument {key}.'					)
					for arg, attribute_name in zip(__UpperCAmelCase			,							self.attributes					):
								if attribute_name in kwargs:
											raise TypeError(f'Got multiple values for argument {attribute_name}.'					)
								else:
											__UpperCAmelCase :   int    					=  arg

					if len(__UpperCAmelCase					) != len(self.attributes					):
								raise ValueError(
								    f'This processor requires {len(self.attributes					)} arguments: {", ".join(self.attributes					)}. Got '
								    f'{len(__UpperCAmelCase					)} arguments instead.'					)

					# Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
					for attribute_name, arg in kwargs.items():
								__UpperCAmelCase :   int    					=  getattr(self			,							f'{attribute_name}_class'					)
								# Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
								__UpperCAmelCase :   str    					=  AUTO_TO_BASE_CLASS_MAPPING.get(__UpperCAmelCase			,							__UpperCAmelCase					)
								if isinstance(__UpperCAmelCase			,							__UpperCAmelCase					):
											__UpperCAmelCase :   str    					=  tuple(getattr(__UpperCAmelCase			,							__UpperCAmelCase					) for n in class_name if n is not None					)
								else:
											__UpperCAmelCase :   Optional[Any]    					=  getattr(__UpperCAmelCase			,							__UpperCAmelCase					)

								if not isinstance(__UpperCAmelCase			,							__UpperCAmelCase					):
											raise ValueError(
											    f'Received a {type(__UpperCAmelCase					).__name__} for argument {attribute_name}, but a {class_name} was expected.'					)

								setattr(self			,							__UpperCAmelCase			,							__UpperCAmelCase					)





		def __repr__(  self					)							->  List[str]:

					'''simple docstring'''
					__UpperCAmelCase :   Any    					=  [f'- {name}: {repr(getattr(self			,							__UpperCAmelCase					)					)}' for name in self.attributes]
					__UpperCAmelCase :   List[Any]    					=  """\n""".join(__UpperCAmelCase					)
					return f'{self.__class__.__name__}:\n{attributes_repr}'





		def 				__A							(  self			,							__UpperCAmelCase			,							__UpperCAmelCase = False			,							**__UpperCAmelCase					)							->  Any:

					'''simple docstring'''
					os.makedirs(__UpperCAmelCase			,							exist_ok=__UpperCAmelCase					)

					if push_to_hub:
								__UpperCAmelCase :   Optional[int]    					=  kwargs.pop("""commit_message"""			,							__UpperCAmelCase					)
								__UpperCAmelCase :   List[Any]    					=  kwargs.pop("""repo_id"""			,							save_directory.split(os.path.sep					)[-1]					)
								__UpperCAmelCase :   Any    					=  self._create_repo(__UpperCAmelCase			,							**__UpperCAmelCase					)
								__UpperCAmelCase :   List[str]    					=  self._get_files_timestamps(__UpperCAmelCase					)
					# If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
					# loaded from the Hub.
					if self._auto_class is not None:
								__UpperCAmelCase :   Union[str, Any]    					=  [getattr(self			,							__UpperCAmelCase					) for attribute_name in self.attributes]
								__UpperCAmelCase :   Dict    					=  [(a.init_kwargs if isinstance(__UpperCAmelCase			,							__UpperCAmelCase					) else a) for a in attrs]
								custom_object_save(self			,							__UpperCAmelCase			,							config=__UpperCAmelCase					)

					for attribute_name in self.attributes:
								__UpperCAmelCase :   Optional[int]    					=  getattr(self			,							__UpperCAmelCase					)
								# Include the processor class in the attribute config so this processor can then be reloaded with the
								# `AutoProcessor` API.
								if hasattr(__UpperCAmelCase			,							"""_set_processor_class"""					):
											attribute._set_processor_class(self.__class__.__name__					)
								attribute.save_pretrained(__UpperCAmelCase					)

					if self._auto_class is not None:
								# We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
								for attribute_name in self.attributes:
											__UpperCAmelCase :   str    					=  getattr(self			,							__UpperCAmelCase					)
											if isinstance(__UpperCAmelCase			,							__UpperCAmelCase					):
														del attribute.init_kwargs["auto_map"]

					if push_to_hub:
								self._upload_modified_files(
								    __UpperCAmelCase			,							__UpperCAmelCase			,							__UpperCAmelCase			,							commit_message=__UpperCAmelCase			,							token=kwargs.get("""use_auth_token"""					)			,							)





		@classmethod
		def 				__A							(  cls			,							__UpperCAmelCase			,							__UpperCAmelCase = None			,							__UpperCAmelCase = False			,							__UpperCAmelCase = False			,							__UpperCAmelCase = None			,							__UpperCAmelCase = "main"			,							**__UpperCAmelCase			,							)							->  Optional[int]:

					'''simple docstring'''
					__UpperCAmelCase :   List[str]    					=  cache_dir
					__UpperCAmelCase :   Tuple    					=  force_download
					__UpperCAmelCase :   Optional[int]    					=  local_files_only
					__UpperCAmelCase :   str    					=  revision

					__UpperCAmelCase :   int    					=  kwargs.pop("""use_auth_token"""			,							__UpperCAmelCase					)
					if use_auth_token is not None:
								warnings.warn(
								    """The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers."""			,							__UpperCAmelCase					)
								if token is not None:
											raise ValueError(
											    """`token` and `use_auth_token` are both specified. Please set only the argument `token`."""					)
								__UpperCAmelCase :   Any    					=  use_auth_token

					if token is not None:
								# change to `token` in a follow-up PR
								__UpperCAmelCase :   Dict    					=  token

					__UpperCAmelCase :   Optional[int]    					=  cls._get_arguments_from_pretrained(__UpperCAmelCase			,							**__UpperCAmelCase					)
					return cls(*__UpperCAmelCase					)





		@classmethod
		def 				__A							(  cls			,							__UpperCAmelCase="AutoProcessor"					)							->  Dict:

					'''simple docstring'''
					if not isinstance(__UpperCAmelCase			,							__UpperCAmelCase					):
								__UpperCAmelCase :   List[Any]    					=  auto_class.__name__

					import transformers.models.auto as auto_module

					if not hasattr(__UpperCAmelCase			,							__UpperCAmelCase					):
								raise ValueError(f'{auto_class} is not a valid auto class.'					)

					__UpperCAmelCase :   Optional[Any]    					=  auto_class





		@classmethod
		def 				__A							(  cls			,							__UpperCAmelCase			,							**__UpperCAmelCase					)							->  List[str]:

					'''simple docstring'''
					__UpperCAmelCase :   Union[str, Any]    					=  []
					for attribute_name in cls.attributes:
								__UpperCAmelCase :   List[Any]    					=  getattr(cls			,							f'{attribute_name}_class'					)
								if isinstance(__UpperCAmelCase			,							__UpperCAmelCase					):
											__UpperCAmelCase :   str    					=  tuple(getattr(__UpperCAmelCase			,							__UpperCAmelCase					) if n is not None else None for n in class_name					)
											__UpperCAmelCase :   int    					=  kwargs.get("""use_fast"""			,							__UpperCAmelCase					)
											if use_fast and classes[1] is not None:
														__UpperCAmelCase :   int    					=  classes[1]
											else:
														__UpperCAmelCase :   Optional[Any]    					=  classes[0]
								else:
											__UpperCAmelCase :   str    					=  getattr(__UpperCAmelCase			,							__UpperCAmelCase					)

								args.append(attribute_class.from_pretrained(__UpperCAmelCase			,							**__UpperCAmelCase					)					)
					return args





		@property
		def 				__A							(  self					)							->  Tuple:

					'''simple docstring'''
					__UpperCAmelCase :   Optional[Any]    					=  getattr(self			,							self.attributes[0]					)
					return getattr(__UpperCAmelCase			,							"""model_input_names"""			,							__UpperCAmelCase					)


_UpperCamelCase									=      copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
					_UpperCamelCase									=      ProcessorMixin.push_to_hub.__doc__.format(
					    object='''processor''', object_class='''AutoProcessor''', object_files='''processor files'''
					)




