


import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_lowerCAmelCase:	str      							=					logging.get_logger(__name__)

_lowerCAmelCase:	Union[str, Any]      							=					{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_lowerCAmelCase:	List[Any]      							=					{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

_lowerCAmelCase:	Union[str, Any]      							=					{
    'openai-gpt': 512,
}






def      _lowercase(				__a							:	int    ):
							a__     						=text.strip()
							if not text:
														return []
							a__     						=text.split()
							return tokens




class 			lowercase_				(lowercase__      ):





				def __init__(							self					,     lowercase_=True					,     lowercase_=None					,     lowercase_=True					,     lowercase_=None					,     lowercase_=True					,     )     ->   int:
											if never_split is None:
																		a__     						=[]
											a__     						=do_lower_case
											a__     						=set(lowercase_)
											a__     						=tokenize_chinese_chars
											a__     						=strip_accents
											a__     						=do_split_on_punc





				def 	__UpperCamelCase						(							self					,     lowercase_					,     lowercase_=None)     ->   List[Any]:
											a__     						=self.never_split.union(set(lowercase_)) if never_split else self.never_split
											a__     						=self._clean_text(lowercase_)

											# This was added on November 1st, 2018 for the multilingual and Chinese
											# models. This is also applied to the English models now, but it doesn't
											# matter since the English models were not trained on any Chinese data
											# and generally don't have any Chinese data in them (there are Chinese
											# characters in the vocabulary because Wikipedia does have some Chinese
											# words in the English Wikipedia.).
											if self.tokenize_chinese_chars:
																		a__     						=self._tokenize_chinese_chars(lowercase_)
											# prevents treating the same character with different unicode codepoints as different characters
											a__     						=unicodedata.normalize('NFC'					,     lowercase_)
											a__     						=whitespace_tokenize(lowercase_)
											a__     						=[]
											for token in orig_tokens:
																		if token not in never_split:
																									if self.do_lower_case:
																																a__     						=token.lower()
																																if self.strip_accents is not False:
																																							a__     						=self._run_strip_accents(lowercase_)
																									elif self.strip_accents:
																																a__     						=self._run_strip_accents(lowercase_)
																		split_tokens.extend(self._run_split_on_punc(lowercase_					,     lowercase_))

											a__     						=whitespace_tokenize(' '.join(lowercase_))
											return output_tokens





				def 	__UpperCamelCase						(							self					,     lowercase_)     ->   int:
											a__     						=unicodedata.normalize('NFD'					,     lowercase_)
											a__     						=[]
											for char in text:
																		a__     						=unicodedata.category(lowercase_)
																		if cat == "Mn":
																									continue
																		output.append(lowercase_)
											return "".join(lowercase_)





				def 	__UpperCamelCase						(							self					,     lowercase_					,     lowercase_=None)     ->   Optional[int]:
											if not self.do_split_on_punc or (never_split is not None and text in never_split):
																		return [text]
											a__     						=list(lowercase_)
											a__     						=0
											a__     						=True
											a__     						=[]
											while i < len(lowercase_):
																		a__     						=chars[i]
																		if _is_punctuation(lowercase_):
																									output.append([char])
																									a__     						=True
																		else:
																									if start_new_word:
																																output.append([])
																									a__     						=False
																									output[-1].append(lowercase_)
																		i += 1

											return ["".join(lowercase_) for x in output]





				def 	__UpperCamelCase						(							self					,     lowercase_)     ->   Any:
											a__     						=[]
											for char in text:
																		a__     						=ord(lowercase_)
																		if self._is_chinese_char(lowercase_):
																									output.append(' ')
																									output.append(lowercase_)
																									output.append(' ')
																		else:
																									output.append(lowercase_)
											return "".join(lowercase_)





				def 	__UpperCamelCase						(							self					,     lowercase_)     ->   Optional[int]:
											# This defines a "chinese character" as anything in the CJK Unicode block:
											#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
											#
											# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
											# despite its name. The modern Korean Hangul alphabet is a different block,
											# as is Japanese Hiragana and Katakana. Those alphabets are used to write
											# space-separated words, so they are not treated specially and handled
											# like the all of the other languages.
											if (
											    (cp >= 0X4E00 and cp <= 0X9FFF)
											    or (cp >= 0X3400 and cp <= 0X4DBF)  #
											    or (cp >= 0X20000 and cp <= 0X2A6DF)  #
											    or (cp >= 0X2A700 and cp <= 0X2B73F)  #
											    or (cp >= 0X2B740 and cp <= 0X2B81F)  #
											    or (cp >= 0X2B820 and cp <= 0X2CEAF)  #
											    or (cp >= 0XF900 and cp <= 0XFAFF)
											    or (cp >= 0X2F800 and cp <= 0X2FA1F)  #
											):  #
																		return True

											return False





				def 	__UpperCamelCase						(							self					,     lowercase_)     ->   Union[str, Any]:
											a__     						=[]
											for char in text:
																		a__     						=ord(lowercase_)
																		if cp == 0 or cp == 0XFFFD or _is_control(lowercase_):
																									continue
																		if _is_whitespace(lowercase_):
																									output.append(' ')
																		else:
																									output.append(lowercase_)
											return "".join(lowercase_)







def      _lowercase(				__a							:	str    ):
							a__     						=set()
							a__     						=word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)    )
														a__     						=char
							return pairs






def      _lowercase(				__a							:	List[Any]    ):
							a__     						=text.replace('—'						,    '-'    )
							a__     						=text.replace('–'						,    '-'    )
							a__     						=text.replace('―'						,    '-'    )
							a__     						=text.replace('…'						,    '...'    )
							a__     						=text.replace('´'						,    '\''    )
							a__     						=re.sub(r'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'						,    r' \1 '						,    __a    )
							a__     						=re.sub(r'\s*\n\s*'						,    ' \n '						,    __a    )
							a__     						=re.sub(r'[^\S\n]+'						,    ' '						,    __a    )
							return text.strip()







class 			lowercase_				(lowercase__      ):
				snake_case  						=VOCAB_FILES_NAMES
				snake_case  						=PRETRAINED_VOCAB_FILES_MAP
				snake_case  						=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				snake_case  						=['input_ids', 'attention_mask']





				def __init__(							self					,     lowercase_					,     lowercase_					,     lowercase_="<unk>"					,     **lowercase_)     ->   Union[str, Any]:
											super().__init__(unk_token=lowercase_					,     **lowercase_)

											try:
																		import ftfy
																		from spacy.lang.en import English

																		a__     						=English()
																		a__     						=_nlp.tokenizer
																		a__     						=ftfy.fix_text
											except ImportError:
																		logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.')
																		a__     						=BasicTokenizer(do_lower_case=lowercase_)
																		a__     						=None

											with open(lowercase_					,     encoding='utf-8') as vocab_handle:
																		a__     						=json.load(lowercase_)
											a__     						={v: k for k, v in self.encoder.items()}
											with open(lowercase_					,     encoding='utf-8') as merges_handle:
																		a__     						=merges_handle.read().split('\n')[1:-1]
											a__     						=[tuple(merge.split()) for merge in merges]
											a__     						=dict(zip(lowercase_					,     range(len(lowercase_))))
											a__     						={}





				@property
				def 	__UpperCamelCase						(							self)     ->   Optional[int]:
											return True





				@property
				def 	__UpperCamelCase						(							self)     ->   Union[str, Any]:
											return len(self.encoder)





				def 	__UpperCamelCase						(							self)     ->   Optional[int]:
											return dict(self.encoder					,     **self.added_tokens_encoder)





				def 	__UpperCamelCase						(							self					,     lowercase_)     ->   Optional[int]:
											a__     						=tuple(token[:-1]) + (token[-1] + '</w>',)
											if token in self.cache:
																		return self.cache[token]
											a__     						=get_pairs(lowercase_)

											if not pairs:
																		return token + "</w>"

											while True:
																		a__     						=min(lowercase_					,     key=lambda lowercase_: self.bpe_ranks.get(lowercase_					,     float('inf')))
																		if bigram not in self.bpe_ranks:
																									break
																		a__						,    a__     						=bigram
																		a__     						=[]
																		a__     						=0
																		while i < len(lowercase_):
																									try:
																																a__     						=word.index(lowercase_					,     lowercase_)
																									except ValueError:
																																new_word.extend(word[i:])
																																break
																									else:
																																new_word.extend(word[i:j])
																																a__     						=j

																									if word[i] == first and i < len(lowercase_) - 1 and word[i + 1] == second:
																																new_word.append(first + second)
																																i += 2
																									else:
																																new_word.append(word[i])
																																i += 1
																		a__     						=tuple(lowercase_)
																		a__     						=new_word
																		if len(lowercase_) == 1:
																									break
																		else:
																									a__     						=get_pairs(lowercase_)
											a__     						=' '.join(lowercase_)
											if word == "\n  </w>":
																		a__     						='\n</w>'
											a__     						=word
											return word





				def 	__UpperCamelCase						(							self					,     lowercase_)     ->   List[Any]:
											a__     						=[]
											if self.fix_text is None:
																		# Using BERT's BasicTokenizer
																		a__     						=self.nlp.tokenize(lowercase_)
																		for token in text:
																									split_tokens.extend(list(self.bpe(lowercase_).split(' ')))
											else:
																		# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
																		a__     						=self.nlp(text_standardize(self.fix_text(lowercase_)))
																		for token in text:
																									split_tokens.extend(list(self.bpe(token.text.lower()).split(' ')))
											return split_tokens





				def 	__UpperCamelCase						(							self					,     lowercase_)     ->   int:
											return self.encoder.get(lowercase_					,     self.encoder.get(self.unk_token))





				def 	__UpperCamelCase						(							self					,     lowercase_)     ->   Dict:
											return self.decoder.get(lowercase_					,     self.unk_token)





				def 	__UpperCamelCase						(							self					,     lowercase_)     ->   Union[str, Any]:
											a__     						=''.join(lowercase_).replace('</w>'					,     ' ').strip()
											return out_string





				def 	__UpperCamelCase						(							self					,     lowercase_					,     lowercase_ = None)     ->   Tuple[str]:
											if not os.path.isdir(lowercase_):
																		logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
																		return
											a__     						=os.path.join(
											    lowercase_					,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
											a__     						=os.path.join(
											    lowercase_					,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

											with open(lowercase_					,     'w'					,     encoding='utf-8') as f:
																		f.write(json.dumps(self.encoder					,     indent=2					,     sort_keys=lowercase_					,     ensure_ascii=lowercase_) + '\n')

											a__     						=0
											with open(lowercase_					,     'w'					,     encoding='utf-8') as writer:
																		writer.write('#version: 0.2\n')
																		for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,     key=lambda lowercase_: kv[1]):
																									if index != token_index:
																																logger.warning(
																																    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																																    ' Please check that the tokenizer is not corrupted!')
																																a__     						=token_index
																									writer.write(' '.join(lowercase_) + '\n')
																									index += 1

											return vocab_file, merge_file





import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


_lowerCAmelCase:	Any      							=					logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
_lowerCAmelCase:	Dict      							=					direct_transformers_import(Path(__file__).parent)


_lowerCAmelCase:	Optional[Any]      							=					{
    'AutoTokenizer': 'PreTrainedTokenizerBase',
    'AutoFeatureExtractor': 'FeatureExtractionMixin',
    'AutoImageProcessor': 'ImageProcessingMixin',
}




class 			lowercase_				(lowercase__      ):
				snake_case  						=['feature_extractor', 'tokenizer']
				# Names need to be attr_class for attr in attributes
				snake_case  						=None
				snake_case  						=None
				snake_case  						=None





				def __init__(							self					,     *lowercase_					,     **lowercase_)     ->   str:
											# Sanitize args and kwargs
											for key in kwargs:
																		if key not in self.attributes:
																									raise TypeError(F"""Unexpected keyword argument {key}.""")
											for arg, attribute_name in zip(lowercase_					,     self.attributes):
																		if attribute_name in kwargs:
																									raise TypeError(F"""Got multiple values for argument {attribute_name}.""")
																		else:
																									a__     						=arg

											if len(lowercase_) != len(self.attributes):
																		raise ValueError(
																		    F"""This processor requires {len(self.attributes)} arguments: {', '.join(self.attributes)}. Got """
																		    F"""{len(lowercase_)} arguments instead.""")

											# Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
											for attribute_name, arg in kwargs.items():
																		a__     						=getattr(self					,     F"""{attribute_name}_class""")
																		# Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
																		a__     						=AUTO_TO_BASE_CLASS_MAPPING.get(lowercase_					,     lowercase_)
																		if isinstance(lowercase_					,     lowercase_):
																									a__     						=tuple(getattr(lowercase_					,     lowercase_) for n in class_name if n is not None)
																		else:
																									a__     						=getattr(lowercase_					,     lowercase_)

																		if not isinstance(lowercase_					,     lowercase_):
																									raise ValueError(
																									    F"""Received a {type(lowercase_).__name__} for argument {attribute_name}, but a {class_name} was expected.""")

																		setattr(self					,     lowercase_					,     lowercase_)





				def __repr__(							self)     ->   List[str]:
											a__     						=[F"""- {name}: {repr(getattr(self					,     lowercase_))}""" for name in self.attributes]
											a__     						='\n'.join(lowercase_)
											return F"""{self.__class__.__name__}:\n{attributes_repr}"""





				def 	__UpperCamelCase						(							self					,     lowercase_					,     lowercase_ = False					,     **lowercase_)     ->   Optional[int]:
											os.makedirs(lowercase_					,     exist_ok=lowercase_)

											if push_to_hub:
																		a__     						=kwargs.pop('commit_message'					,     lowercase_)
																		a__     						=kwargs.pop('repo_id'					,     save_directory.split(os.path.sep)[-1])
																		a__     						=self._create_repo(lowercase_					,     **lowercase_)
																		a__     						=self._get_files_timestamps(lowercase_)
											# If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
											# loaded from the Hub.
											if self._auto_class is not None:
																		a__     						=[getattr(self					,     lowercase_) for attribute_name in self.attributes]
																		a__     						=[(a.init_kwargs if isinstance(lowercase_					,     lowercase_) else a) for a in attrs]
																		custom_object_save(self					,     lowercase_					,     config=lowercase_)

											for attribute_name in self.attributes:
																		a__     						=getattr(self					,     lowercase_)
																		# Include the processor class in the attribute config so this processor can then be reloaded with the
																		# `AutoProcessor` API.
																		if hasattr(lowercase_					,     '_set_processor_class'):
																									attribute._set_processor_class(self.__class__.__name__)
																		attribute.save_pretrained(lowercase_)

											if self._auto_class is not None:
																		# We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
																		for attribute_name in self.attributes:
																									a__     						=getattr(self					,     lowercase_)
																									if isinstance(lowercase_					,     lowercase_):
																																del attribute.init_kwargs["auto_map"]

											if push_to_hub:
																		self._upload_modified_files(
																		    lowercase_					,     lowercase_					,     lowercase_					,     commit_message=lowercase_					,     token=kwargs.get('use_auth_token')					,     )





				@classmethod
				def 	__UpperCamelCase						(							cls					,     lowercase_					,     lowercase_ = None					,     lowercase_ = False					,     lowercase_ = False					,     lowercase_ = None					,     lowercase_ = "main"					,     **lowercase_					,     )     ->   str:
											a__     						=cache_dir
											a__     						=force_download
											a__     						=local_files_only
											a__     						=revision

											a__     						=kwargs.pop('use_auth_token'					,     lowercase_)
											if use_auth_token is not None:
																		warnings.warn(
																		    'The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.'					,     lowercase_)
																		if token is not None:
																									raise ValueError(
																									    '`token` and `use_auth_token` are both specified. Please set only the argument `token`.')
																		a__     						=use_auth_token

											if token is not None:
																		# change to `token` in a follow-up PR
																		a__     						=token

											a__     						=cls._get_arguments_from_pretrained(lowercase_					,     **lowercase_)
											return cls(*lowercase_)





				@classmethod
				def 	__UpperCamelCase						(							cls					,     lowercase_="AutoProcessor")     ->   Any:
											if not isinstance(lowercase_					,     lowercase_):
																		a__     						=auto_class.__name__

											import transformers.models.auto as auto_module

											if not hasattr(lowercase_					,     lowercase_):
																		raise ValueError(F"""{auto_class} is not a valid auto class.""")

											a__     						=auto_class





				@classmethod
				def 	__UpperCamelCase						(							cls					,     lowercase_					,     **lowercase_)     ->   Optional[int]:
											a__     						=[]
											for attribute_name in cls.attributes:
																		a__     						=getattr(cls					,     F"""{attribute_name}_class""")
																		if isinstance(lowercase_					,     lowercase_):
																									a__     						=tuple(getattr(lowercase_					,     lowercase_) if n is not None else None for n in class_name)
																									a__     						=kwargs.get('use_fast'					,     lowercase_)
																									if use_fast and classes[1] is not None:
																																a__     						=classes[1]
																									else:
																																a__     						=classes[0]
																		else:
																									a__     						=getattr(lowercase_					,     lowercase_)

																		args.append(attribute_class.from_pretrained(lowercase_					,     **lowercase_))
											return args





				@property
				def 	__UpperCamelCase						(							self)     ->   List[Any]:
											a__     						=getattr(self					,     self.attributes[0])
											return getattr(lowercase_					,     'model_input_names'					,     lowercase_)







_lowerCAmelCase:	Dict      							=					copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
	_lowerCAmelCase:	Tuple      							=					ProcessorMixin.push_to_hub.__doc__.format(
	    object='processor', object_class='AutoProcessor', object_files='processor files'
	)

