



"""simple docstring"""

import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


A_	:						Tuple				=			logging.get_logger(__name__)

A_	:						List[str]				=			{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

A_	:						List[Any]				=			{
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

A_	:						Dict				=			{
    'openai-gpt': 512,
}
def    __snake_case       (  __A			:	Any    )    ->					int:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    text.strip()
				if not text:
								return []
				SCREAMING_SNAKE_CASE	:       List[Any] 							=    text.split()
				return tokens
class 						lowerCAmelCase__	(  _lowerCamelCase   ):


	'''simple docstring'''







	def __init__(			self							:					Union[str, Any]    ,					_SCREAMING_SNAKE_CASE							:					List[Any]=True    ,					_SCREAMING_SNAKE_CASE							:					Union[str, Any]=None    ,					_SCREAMING_SNAKE_CASE							:					Dict=True    ,					_SCREAMING_SNAKE_CASE							:					str=None    ,					_SCREAMING_SNAKE_CASE							:					Tuple=True    ,					)	->  Dict:





					"""simple docstring"""


					if never_split is None:
									SCREAMING_SNAKE_CASE	:       Optional[int] 							=    []
					SCREAMING_SNAKE_CASE	:       str 							=    do_lower_case
					SCREAMING_SNAKE_CASE	:       List[str] 							=    set(_SCREAMING_SNAKE_CASE						)
					SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    tokenize_chinese_chars
					SCREAMING_SNAKE_CASE	:       int 							=    strip_accents
					SCREAMING_SNAKE_CASE	:       Any 							=    do_split_on_punc





	def  _lowerCAmelCase    (			self							:					Any    ,					_SCREAMING_SNAKE_CASE							:					Union[str, Any]    ,					_SCREAMING_SNAKE_CASE							:					List[str]=None						)	->  Tuple:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    self.never_split.union(set(_SCREAMING_SNAKE_CASE						)						) if never_split else self.never_split
					SCREAMING_SNAKE_CASE	:       Any 							=    self._clean_text(_SCREAMING_SNAKE_CASE						)

					# This was added on November 1st, 2018 for the multilingual and Chinese
					# models. This is also applied to the English models now, but it doesn't
					# matter since the English models were not trained on any Chinese data
					# and generally don't have any Chinese data in them (there are Chinese
					# characters in the vocabulary because Wikipedia does have some Chinese
					# words in the English Wikipedia.).
					if self.tokenize_chinese_chars:
									SCREAMING_SNAKE_CASE	:       int 							=    self._tokenize_chinese_chars(_SCREAMING_SNAKE_CASE						)
					# prevents treating the same character with different unicode codepoints as different characters
					SCREAMING_SNAKE_CASE	:       List[str] 							=    unicodedata.normalize('NFC'    ,					_SCREAMING_SNAKE_CASE						)
					SCREAMING_SNAKE_CASE	:       List[Any] 							=    whitespace_tokenize(_SCREAMING_SNAKE_CASE						)
					SCREAMING_SNAKE_CASE	:       str 							=    []
					for token in orig_tokens:
									if token not in never_split:
													if self.do_lower_case:
																	SCREAMING_SNAKE_CASE	:       int 							=    token.lower()
																	if self.strip_accents is not False:
																					SCREAMING_SNAKE_CASE	:       Any 							=    self._run_strip_accents(_SCREAMING_SNAKE_CASE						)
													elif self.strip_accents:
																	SCREAMING_SNAKE_CASE	:       Optional[int] 							=    self._run_strip_accents(_SCREAMING_SNAKE_CASE						)
									split_tokens.extend(self._run_split_on_punc(_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE						)						)

					SCREAMING_SNAKE_CASE	:       Dict 							=    whitespace_tokenize(' '.join(_SCREAMING_SNAKE_CASE						)						)
					return output_tokens





	def  _lowerCAmelCase    (			self							:					List[str]    ,					_SCREAMING_SNAKE_CASE							:					List[Any]						)	->  List[str]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    unicodedata.normalize('NFD'    ,					_SCREAMING_SNAKE_CASE						)
					SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    []
					for char in text:
									SCREAMING_SNAKE_CASE	:       Optional[int] 							=    unicodedata.category(_SCREAMING_SNAKE_CASE						)
									if cat == "Mn":
													continue
									output.append(_SCREAMING_SNAKE_CASE						)
					return "".join(_SCREAMING_SNAKE_CASE						)





	def  _lowerCAmelCase    (			self							:					Optional[Any]    ,					_SCREAMING_SNAKE_CASE							:					Optional[Any]    ,					_SCREAMING_SNAKE_CASE							:					Optional[int]=None						)	->  Union[str, Any]:





					"""simple docstring"""


					if not self.do_split_on_punc or (never_split is not None and text in never_split):
									return [text]
					SCREAMING_SNAKE_CASE	:       List[Any] 							=    list(_SCREAMING_SNAKE_CASE						)
					SCREAMING_SNAKE_CASE	:       List[Any] 							=    0
					SCREAMING_SNAKE_CASE	:       Tuple 							=    True
					SCREAMING_SNAKE_CASE	:       Dict 							=    []
					while i < len(_SCREAMING_SNAKE_CASE						):
									SCREAMING_SNAKE_CASE	:       Optional[int] 							=    chars[i]
									if _is_punctuation(_SCREAMING_SNAKE_CASE						):
													output.append([char]						)
													SCREAMING_SNAKE_CASE	:       int 							=    True
									else:
													if start_new_word:
																	output.append([]						)
													SCREAMING_SNAKE_CASE	:       List[Any] 							=    False
													output[-1].append(_SCREAMING_SNAKE_CASE						)
									i += 1

					return ["".join(_SCREAMING_SNAKE_CASE						) for x in output]





	def  _lowerCAmelCase    (			self							:					Optional[int]    ,					_SCREAMING_SNAKE_CASE							:					List[str]						)	->  Optional[Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Dict 							=    []
					for char in text:
									SCREAMING_SNAKE_CASE	:       int 							=    ord(_SCREAMING_SNAKE_CASE						)
									if self._is_chinese_char(_SCREAMING_SNAKE_CASE						):
													output.append(' '						)
													output.append(_SCREAMING_SNAKE_CASE						)
													output.append(' '						)
									else:
													output.append(_SCREAMING_SNAKE_CASE						)
					return "".join(_SCREAMING_SNAKE_CASE						)





	def  _lowerCAmelCase    (			self							:					List[str]    ,					_SCREAMING_SNAKE_CASE							:					Dict						)	->  List[Any]:





					"""simple docstring"""


					if (
					    (cp >= 0x4e00 and cp <= 0x9fff)
					    or (cp >= 0x3400 and cp <= 0x4dbf)  #
					    or (cp >= 0x2_0000 and cp <= 0x2_a6df)  #
					    or (cp >= 0x2_a700 and cp <= 0x2_b73f)  #
					    or (cp >= 0x2_b740 and cp <= 0x2_b81f)  #
					    or (cp >= 0x2_b820 and cp <= 0x2_ceaf)  #
					    or (cp >= 0xf900 and cp <= 0xfaff)
					    or (cp >= 0x2_f800 and cp <= 0x2_fa1f)  #
					):  #
									return True

					return False







	def  _lowerCAmelCase    (			self							:					Optional[int]    ,					_SCREAMING_SNAKE_CASE							:					List[Any]						)	->  Optional[Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Dict 							=    []
					for char in text:
									SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    ord(_SCREAMING_SNAKE_CASE						)
									if cp == 0 or cp == 0xfffd or _is_control(_SCREAMING_SNAKE_CASE						):
													continue
									if _is_whitespace(_SCREAMING_SNAKE_CASE						):
													output.append(' '						)
									else:
													output.append(_SCREAMING_SNAKE_CASE						)
					return "".join(_SCREAMING_SNAKE_CASE						)




def    __snake_case       (  __A			:	Tuple    )    ->					Union[str, Any]:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE	:       List[str] 							=    set()
				SCREAMING_SNAKE_CASE	:       List[str] 							=    word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)    )
								SCREAMING_SNAKE_CASE	:       Any 							=    char
				return pairs
def    __snake_case       (  __A			:	Any    )    ->					List[str]:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    text.replace('—'       ,		'-'    )
				SCREAMING_SNAKE_CASE	:       Optional[int] 							=    text.replace('–'       ,		'-'    )
				SCREAMING_SNAKE_CASE	:       Optional[int] 							=    text.replace('―'       ,		'-'    )
				SCREAMING_SNAKE_CASE	:       Any 							=    text.replace('…'       ,		'...'    )
				SCREAMING_SNAKE_CASE	:       int 							=    text.replace('´'       ,		'\''    )
				SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    re.sub(r'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'       ,		r' \1 '       ,		__A    )
				SCREAMING_SNAKE_CASE	:       str 							=    re.sub(r'\s*\n\s*'       ,		' \n '       ,		__A    )
				SCREAMING_SNAKE_CASE	:       Any 							=    re.sub(r'[^\S\n]+'       ,		' '       ,		__A    )
				return text.strip()







class 						lowerCAmelCase__	(  _lowerCamelCase   ):


	'''simple docstring'''







	_SCREAMING_SNAKE_CASE  :		Tuple     	=   VOCAB_FILES_NAMES
	_SCREAMING_SNAKE_CASE  :		List[str]     	=   PRETRAINED_VOCAB_FILES_MAP
	_SCREAMING_SNAKE_CASE  :		List[Any]     	=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_SCREAMING_SNAKE_CASE  :		Tuple     	=   ['''input_ids''', '''attention_mask''']





	def __init__(			self							:					List[Any]    ,					_SCREAMING_SNAKE_CASE							:					List[str]    ,					_SCREAMING_SNAKE_CASE							:					List[str]    ,					_SCREAMING_SNAKE_CASE							:					Optional[int]="<unk>"    ,					**_SCREAMING_SNAKE_CASE							:					Any						)	->  int:





					"""simple docstring"""


					super().__init__(unk_token=_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE						)

					try:
									import ftfy
									from spacy.lang.en import English

									SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    English()
									SCREAMING_SNAKE_CASE	:       Any 							=    _nlp.tokenizer
									SCREAMING_SNAKE_CASE	:       int 							=    ftfy.fix_text
					except ImportError:
									logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'						)
									SCREAMING_SNAKE_CASE	:       List[Any] 							=    BasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE						)
									SCREAMING_SNAKE_CASE	:       Tuple 							=    None

					with open(_SCREAMING_SNAKE_CASE    ,					encoding='utf-8'						) as vocab_handle:
									SCREAMING_SNAKE_CASE	:       int 							=    json.load(_SCREAMING_SNAKE_CASE						)
					SCREAMING_SNAKE_CASE	:       List[Any] 							=    {v: k for k, v in self.encoder.items()}
					with open(_SCREAMING_SNAKE_CASE    ,					encoding='utf-8'						) as merges_handle:
									SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    merges_handle.read().split('\n'						)[1:-1]
					SCREAMING_SNAKE_CASE	:       int 							=    [tuple(merge.split()						) for merge in merges]
					SCREAMING_SNAKE_CASE	:       List[str] 							=    dict(zip(_SCREAMING_SNAKE_CASE    ,					range(len(_SCREAMING_SNAKE_CASE						)						)						)						)
					SCREAMING_SNAKE_CASE	:       Dict 							=    {}





	@property
	def  _lowerCAmelCase    (			self							:					List[str]						)	->  List[Any]:





					"""simple docstring"""


					return True





	@property
	def  _lowerCAmelCase    (			self							:					Tuple						)	->  Any:





					"""simple docstring"""


					return len(self.encoder						)





	def  _lowerCAmelCase    (			self							:					Any						)	->  Optional[Any]:





					"""simple docstring"""


					return dict(self.encoder    ,					**self.added_tokens_encoder						)





	def  _lowerCAmelCase    (			self							:					Union[str, Any]    ,					_SCREAMING_SNAKE_CASE							:					int						)	->  List[Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Optional[int] 							=    tuple(token[:-1]						) + (token[-1] + '</w>',)
					if token in self.cache:
									return self.cache[token]
					SCREAMING_SNAKE_CASE	:       Optional[int] 							=    get_pairs(_SCREAMING_SNAKE_CASE						)

					if not pairs:
									return token + "</w>"

					while True:
									SCREAMING_SNAKE_CASE	:       Dict 							=    min(_SCREAMING_SNAKE_CASE    ,					key=lambda _SCREAMING_SNAKE_CASE						: self.bpe_ranks.get(_SCREAMING_SNAKE_CASE    ,					float('inf'						)						)						)
									if bigram not in self.bpe_ranks:
													break
									SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE	:       int 							=    bigram
									SCREAMING_SNAKE_CASE	:       Optional[int] 							=    []
									SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    0
									while i < len(_SCREAMING_SNAKE_CASE						):
													try:
																	SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    word.index(_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE						)
													except ValueError:
																	new_word.extend(word[i:]						)
																	break
													else:
																	new_word.extend(word[i:j]						)
																	SCREAMING_SNAKE_CASE	:       Tuple 							=    j

													if word[i] == first and i < len(_SCREAMING_SNAKE_CASE						) - 1 and word[i + 1] == second:
																	new_word.append(first + second						)
																	i += 2
													else:
																	new_word.append(word[i]						)
																	i += 1
									SCREAMING_SNAKE_CASE	:       int 							=    tuple(_SCREAMING_SNAKE_CASE						)
									SCREAMING_SNAKE_CASE	:       List[str] 							=    new_word
									if len(_SCREAMING_SNAKE_CASE						) == 1:
													break
									else:
													SCREAMING_SNAKE_CASE	:       int 							=    get_pairs(_SCREAMING_SNAKE_CASE						)
					SCREAMING_SNAKE_CASE	:       List[str] 							=    ' '.join(_SCREAMING_SNAKE_CASE						)
					if word == "\n  </w>":
									SCREAMING_SNAKE_CASE	:       Any 							=    '\n</w>'
					SCREAMING_SNAKE_CASE	:       Optional[int] 							=    word
					return word





	def  _lowerCAmelCase    (			self							:					List[str]    ,					_SCREAMING_SNAKE_CASE							:					Dict						)	->  Optional[int]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    []
					if self.fix_text is None:
									# Using BERT's BasicTokenizer
									SCREAMING_SNAKE_CASE	:       Optional[int] 							=    self.nlp.tokenize(_SCREAMING_SNAKE_CASE						)
									for token in text:
													split_tokens.extend(list(self.bpe(_SCREAMING_SNAKE_CASE						).split(' '						)						)						)
					else:
									# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
									SCREAMING_SNAKE_CASE	:       Optional[int] 							=    self.nlp(text_standardize(self.fix_text(_SCREAMING_SNAKE_CASE						)						)						)
									for token in text:
													split_tokens.extend(list(self.bpe(token.text.lower()						).split(' '						)						)						)
					return split_tokens





	def  _lowerCAmelCase    (			self							:					Any    ,					_SCREAMING_SNAKE_CASE							:					int						)	->  Any:





					"""simple docstring"""


					return self.encoder.get(_SCREAMING_SNAKE_CASE    ,					self.encoder.get(self.unk_token						)						)





	def  _lowerCAmelCase    (			self							:					Optional[Any]    ,					_SCREAMING_SNAKE_CASE							:					List[Any]						)	->  List[str]:





					"""simple docstring"""


					return self.decoder.get(_SCREAMING_SNAKE_CASE    ,					self.unk_token						)





	def  _lowerCAmelCase    (			self							:					int    ,					_SCREAMING_SNAKE_CASE							:					str						)	->  List[Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       str 							=    ''.join(_SCREAMING_SNAKE_CASE						).replace('</w>'    ,					' '						).strip()
					return out_string







	def  _lowerCAmelCase    (			self							:					Any    ,					_SCREAMING_SNAKE_CASE							:					str    ,					_SCREAMING_SNAKE_CASE							:					Optional[str] = None						)	->  Tuple[str]:





					"""simple docstring"""


					if not os.path.isdir(_SCREAMING_SNAKE_CASE						):
									logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
									return
					SCREAMING_SNAKE_CASE	:       Dict 							=    os.path.join(
					    _SCREAMING_SNAKE_CASE    ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)
					SCREAMING_SNAKE_CASE	:       int 							=    os.path.join(
					    _SCREAMING_SNAKE_CASE    ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']						)

					with open(_SCREAMING_SNAKE_CASE    ,					'w'    ,					encoding='utf-8'						) as f:
									f.write(json.dumps(self.encoder    ,					indent=2    ,					sort_keys=_SCREAMING_SNAKE_CASE    ,					ensure_ascii=_SCREAMING_SNAKE_CASE						) + '\n'						)

					SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    0
					with open(_SCREAMING_SNAKE_CASE    ,					'w'    ,					encoding='utf-8'						) as writer:
									writer.write('#version: 0.2\n'						)
									for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,					key=lambda _SCREAMING_SNAKE_CASE						: kv[1]						):
													if index != token_index:
																	logger.warning(
																	    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																	    ' Please check that the tokenizer is not corrupted!'						)
																	SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    token_index
													writer.write(' '.join(_SCREAMING_SNAKE_CASE						) + '\n'						)
													index += 1

					return vocab_file, merge_file










"""simple docstring"""

def    __snake_case       (  __A			:	int    )    ->					int:






				'''simple docstring'''
				if not isinstance(__A       ,		__A    ):
								raise ValueError('double_factorial() only accepts integral values'    )
				if num < 0:
								raise ValueError('double_factorial() not defined for negative values'    )
				SCREAMING_SNAKE_CASE	:       List[str] 							=    1
				for i in range(__A       ,		0       ,		-2    ):
								value *= i
				return value


if __name__ == "__main__":
							import doctest

							doctest.testmod()





