code: 
  import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
  
  
  UpperCamelCase_	:    Dict  					=       logging.get_logger(__name__)
  
  UpperCamelCase_	:    Any  					=       {
      """vocab_file""": """vocab.json""",
      """merges_file""": """merges.txt""",
  }
  
  UpperCamelCase_	:    Optional[int]  					=       {
      """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
      """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
  }
  
  UpperCamelCase_	:    Tuple  					=       {
      """openai-gpt""": 512,
  }
  
  
  
  def 		UpperCamelCase    (					_UpperCAmelCase     :      Union[str, Any]     )      -> List[str]:
  
  
  
  
  
  
  						'''simple docstring'''
  
  						_lowercase					:  List[str]    			=      text.strip()
  						if not text:
  												return []
  						_lowercase					:  List[Any]    			=      text.split()
  						return tokens
  
  
  
  
  class      __lowercase	(	__snake_case					):
  
  
  
  
  
  
  	def __init__(self							:   Dict			,  snake_case							:   Tuple=True			,  snake_case							:   str=None			,  snake_case							:   List[Any]=True			,  snake_case							:   str=None			,  snake_case							:   Tuple=True			,  )   -> Optional[Any]:
  
  
  
  
  
  
  							if never_split is None:
  													_lowercase					:  Tuple    			=      []
  							_lowercase					:  str    			=      do_lower_case
  							_lowercase					:  Optional[Any]    			=      set(snake_case						)
  							_lowercase					:  Dict    			=      tokenize_chinese_chars
  							_lowercase					:  str    			=      strip_accents
  							_lowercase					:  int    			=      do_split_on_punc
  
  
  
  
  
  
  	def 			_a(self							:   str			,  snake_case							:   Union[str, Any]			,  snake_case							:   Optional[Any]=None						)   -> Optional[int]:
  							_lowercase					:  int    			=      self.never_split.union(set(snake_case						)						) if never_split else self.never_split
  							_lowercase					:  Tuple    			=      self._clean_text(snake_case						)
  
  							# This was added on November 1st, 2018 for the multilingual and Chinese
  							# models. This is also applied to the English models now, but it doesn't
  							# matter since the English models were not trained on any Chinese data
  							# and generally don't have any Chinese data in them (there are Chinese
  							# characters in the vocabulary because Wikipedia does have some Chinese
  							# words in the English Wikipedia.).
  							if self.tokenize_chinese_chars:
  													_lowercase					:  int    			=      self._tokenize_chinese_chars(snake_case						)
  							# prevents treating the same character with different unicode codepoints as different characters
  							_lowercase					:  List[Any]    			=      unicodedata.normalize("NFC"			,  snake_case						)
  							_lowercase					:  Tuple    			=      whitespace_tokenize(snake_case						)
  							_lowercase					:  Union[str, Any]    			=      []
  							for token in orig_tokens:
  													if token not in never_split:
  																			if self.do_lower_case:
  																									_lowercase					:  Optional[int]    			=      token.lower()
  																									if self.strip_accents is not False:
  																															_lowercase					:  Optional[int]    			=      self._run_strip_accents(snake_case						)
  																			elif self.strip_accents:
  																									_lowercase					:  List[str]    			=      self._run_strip_accents(snake_case						)
  													split_tokens.extend(self._run_split_on_punc(snake_case			,  snake_case						)						)
  
  							_lowercase					:  Optional[Any]    			=      whitespace_tokenize(" ".join(snake_case						)						)
  							return output_tokens
  
  
  
  
  
  
  	def 			_a(self							:   List[str]			,  snake_case							:   Optional[int]						)   -> List[Any]:
  							_lowercase					:  Tuple    			=      unicodedata.normalize("NFD"			,  snake_case						)
  							_lowercase					:  List[Any]    			=      []
  							for char in text:
  													_lowercase					:  Tuple    			=      unicodedata.category(snake_case						)
  													if cat == "Mn":
  																			continue
  													output.append(snake_case						)
  							return "".join(snake_case						)
  
  
  
  
  
  
  	def 			_a(self							:   List[Any]			,  snake_case							:   str			,  snake_case							:   List[Any]=None						)   -> Union[str, Any]:
  
  
  
  
  
  
  							if not self.do_split_on_punc or (never_split is not None and text in never_split):
  													return [text]
  							_lowercase					:  Union[str, Any]    			=      list(snake_case						)
  							_lowercase					:  Optional[Any]    			=      0
  							_lowercase					:  Optional[int]    			=      True
  							_lowercase					:  Any    			=      []
  							while i < len(snake_case						):
  													_lowercase					:  str    			=      chars[i]
  													if _is_punctuation(snake_case						):
  																			output.append([char]						)
  																			_lowercase					:  Any    			=      True
  													else:
  																			if start_new_word:
  																									output.append([]						)
  																			_lowercase					:  Dict    			=      False
  																			output[-1].append(snake_case						)
  													i += 1
  
  							return ["".join(snake_case						) for x in output]
  
  
  
  
  
  
  	def 			_a(self							:   Dict			,  snake_case							:   Optional[Any]						)   -> Any:
  							_lowercase					:  Optional[int]    			=      []
  							for char in text:
  													_lowercase					:  Tuple    			=      ord(snake_case						)
  													if self._is_chinese_char(snake_case						):
  																			output.append(" "						)
  																			output.append(snake_case						)
  																			output.append(" "						)
  													else:
  																			output.append(snake_case						)
  							return "".join(snake_case						)
  
  
  
  
  
  
  	def 			_a(self							:   int			,  snake_case							:   List[Any]						)   -> Tuple:
  
  
  
  
  
  
  							if (
  							    (cp >= 0X4_e_0_0 and cp <= 0X9_f_f_f)
  							    or (cp >= 0X3_4_0_0 and cp <= 0X4_d_b_f)  #
  							    or (cp >= 0X2_0_0_0_0 and cp <= 0X2_a_6_d_f)  #
  							    or (cp >= 0X2_a_7_0_0 and cp <= 0X2_b_7_3_f)  #
  							    or (cp >= 0X2_b_7_4_0 and cp <= 0X2_b_8_1_f)  #
  							    or (cp >= 0X2_b_8_2_0 and cp <= 0X2_c_e_a_f)  #
  							    or (cp >= 0Xf_9_0_0 and cp <= 0Xf_a_f_f)
  							    or (cp >= 0X2_f_8_0_0 and cp <= 0X2_f_a_1_f)  #
  							):  #
  													return True
  
  							return False
  
  
  
  
  
  
  	def 			_a(self							:   Tuple			,  snake_case							:   Union[str, Any]						)   -> Dict:
  							_lowercase					:  Tuple    			=      []
  							for char in text:
  													_lowercase					:  Union[str, Any]    			=      ord(snake_case						)
  													if cp == 0 or cp == 0Xf_f_f_d or _is_control(snake_case						):
  																			continue
  													if _is_whitespace(snake_case						):
  																			output.append(" "						)
  													else:
  																			output.append(snake_case						)
  							return "".join(snake_case						)
  
  
  
  def 		UpperCamelCase    (					_UpperCAmelCase     :      int     )      -> Dict:
  
  
  
  
  
  
  						'''simple docstring'''
  
  						_lowercase					:  int    			=      set()
  						_lowercase					:  Dict    			=      word[0]
  						for char in word[1:]:
  												pairs.add((prev_char, char)     )
  												_lowercase					:  List[str]    			=      char
  						return pairs
  
  
  
  def 		UpperCamelCase    (					_UpperCAmelCase     :      Union[str, Any]     )      -> Union[str, Any]:
  
  
  
  
  
  
  						'''simple docstring'''
  
  						_lowercase					:  List[Any]    			=      text.replace("—"   , "-"     )
  						_lowercase					:  Optional[Any]    			=      text.replace("–"   , "-"     )
  						_lowercase					:  List[Any]    			=      text.replace("―"   , "-"     )
  						_lowercase					:  Any    			=      text.replace("…"   , "..."     )
  						_lowercase					:  str    			=      text.replace("´"   , "'"     )
  						_lowercase					:  str    			=      re.sub(R"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"   , R" \1 "   , _UpperCAmelCase     )
  						_lowercase					:  Union[str, Any]    			=      re.sub(R"\s*\n\s*"   , " \n "   , _UpperCAmelCase     )
  						_lowercase					:  Any    			=      re.sub(R"[^\S\n]+"   , " "   , _UpperCAmelCase     )
  						return text.strip()
  
  
  
  
  class      __lowercase	(	__snake_case					):
  	_A   			=      VOCAB_FILES_NAMES
  	_A   			=      PRETRAINED_VOCAB_FILES_MAP
  	_A   			=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  	_A   			=      ["input_ids", "attention_mask"]
  
  
  
  
  
  
  	def __init__(self							:   str			,  snake_case							:   str			,  snake_case							:   Any			,  snake_case							:   Dict="<unk>"			,  **snake_case							:   str						)   -> Dict:
  							super().__init__(unk_token=snake_case			,  **snake_case						)
  
  							try:
  													import ftfy
  													from spacy.lang.en import English
  
  													_lowercase					:  List[Any]    			=      English()
  													_lowercase					:  Optional[int]    			=      _nlp.tokenizer
  													_lowercase					:  Tuple    			=      ftfy.fix_text
  							except ImportError:
  													logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."						)
  													_lowercase					:  Dict    			=      BasicTokenizer(do_lower_case=snake_case						)
  													_lowercase					:  str    			=      None
  
  							with open(snake_case			,  encoding="utf-8"						) as vocab_handle:
  													_lowercase					:  str    			=      json.load(snake_case						)
  							_lowercase					:  Union[str, Any]    			=      {v: k for k, v in self.encoder.items()}
  							with open(snake_case			,  encoding="utf-8"						) as merges_handle:
  													_lowercase					:  Dict    			=      merges_handle.read().split("\n"						)[1:-1]
  							_lowercase					:  Dict    			=      [tuple(merge.split()						) for merge in merges]
  							_lowercase					:  str    			=      dict(zip(snake_case			,  range(len(snake_case						)						)						)						)
  							_lowercase					:  List[Any]    			=      {}
  
  
  
  
  
  
  	@property
  	def 			_a(self							:   Dict						)   -> Tuple:
  							return True
  
  
  
  
  
  
  	@property
  	def 			_a(self							:   Optional[int]						)   -> List[Any]:
  							return len(self.encoder						)
  
  
  
  
  
  
  	def 			_a(self							:   Tuple						)   -> int:
  							return dict(self.encoder			,  **self.added_tokens_encoder						)
  
  
  
  
  
  
  	def 			_a(self							:   Dict			,  snake_case							:   List[Any]						)   -> List[str]:
  							_lowercase					:  List[str]    			=      tuple(token[:-1]						) + (token[-1] + "</w>",)
  							if token in self.cache:
  													return self.cache[token]
  							_lowercase					:  Any    			=      get_pairs(snake_case						)
  
  							if not pairs:
  													return token + "</w>"
  
  							while True:
  													_lowercase					:  List[Any]    			=      min(snake_case			,  key=lambda snake_case						: self.bpe_ranks.get(snake_case			,  float("inf"						)						)						)
  													if bigram not in self.bpe_ranks:
  																			break
  													_lowercase					:  List[str]    			=      bigram
  													_lowercase					:  Union[str, Any]    			=      []
  													_lowercase					:  Optional[int]    			=      0
  													while i < len(snake_case						):
  																			try:
  																									_lowercase					:  List[Any]    			=      word.index(snake_case			,  snake_case						)
  																			except ValueError:
  																									new_word.extend(word[i:]						)
  																									break
  																			else:
  																									new_word.extend(word[i:j]						)
  																									_lowercase					:  Any    			=      j
  
  																			if word[i] == first and i < len(snake_case						) - 1 and word[i + 1] == second:
  																									new_word.append(first + second						)
  																									i += 2
  																			else:
  																									new_word.append(word[i]						)
  																									i += 1
  													_lowercase					:  List[Any]    			=      tuple(snake_case						)
  													_lowercase					:  Optional[Any]    			=      new_word
  													if len(snake_case						) == 1:
  																			break
  													else:
  																			_lowercase					:  Optional[Any]    			=      get_pairs(snake_case						)
  							_lowercase					:  List[str]    			=      " ".join(snake_case						)
  							if word == "\n  </w>":
  													_lowercase					:  Tuple    			=      "\n</w>"
  							_lowercase					:  Tuple    			=      word
  							return word
  
  
  
  
  
  
  	def 			_a(self							:   List[str]			,  snake_case							:   Union[str, Any]						)   -> Union[str, Any]:
  							_lowercase					:  str    			=      []
  							if self.fix_text is None:
  													# Using BERT's BasicTokenizer
  													_lowercase					:  Any    			=      self.nlp.tokenize(snake_case						)
  													for token in text:
  																			split_tokens.extend(list(self.bpe(snake_case						).split(" "						)						)						)
  							else:
  													# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
  													_lowercase					:  Union[str, Any]    			=      self.nlp(text_standardize(self.fix_text(snake_case						)						)						)
  													for token in text:
  																			split_tokens.extend(list(self.bpe(token.text.lower()						).split(" "						)						)						)
  							return split_tokens
  
  
  
  
  
  
  	def 			_a(self							:   List[Any]			,  snake_case							:   Optional[int]						)   -> Union[str, Any]:
  							return self.encoder.get(snake_case			,  self.encoder.get(self.unk_token						)						)
  
  
  
  
  
  
  	def 			_a(self							:   Tuple			,  snake_case							:   Any						)   -> List[Any]:
  							return self.decoder.get(snake_case			,  self.unk_token						)
  
  
  
  
  
  
  	def 			_a(self							:   int			,  snake_case							:   List[str]						)   -> List[str]:
  							_lowercase					:  Union[str, Any]    			=      "".join(snake_case						).replace("</w>"			,  " "						).strip()
  							return out_string
  
  
  
  
  
  
  	def 			_a(self							:   Tuple			,  snake_case							:   Tuple			,  snake_case							:   Tuple = None						)   -> Tuple[str]:
  
  
  
  
  
  
  							if not os.path.isdir(snake_case						):
  													logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""						)
  													return
  							_lowercase					:  str    			=      os.path.join(
  							    snake_case			,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
  							_lowercase					:  Any    			=      os.path.join(
  							    snake_case			,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]						)
  
  							with open(snake_case			,  "w"			,  encoding="utf-8"						) as f:
  													f.write(json.dumps(self.encoder			,  indent=2			,  sort_keys=snake_case			,  ensure_ascii=snake_case						) + "\n"						)
  
  							_lowercase					:  int    			=      0
  							with open(snake_case			,  "w"			,  encoding="utf-8"						) as writer:
  													writer.write("#version: 0.2\n"						)
  													for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,  key=lambda snake_case						: kv[1]						):
  																			if index != token_index:
  																									logger.warning(
  																									    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
  																									    " Please check that the tokenizer is not corrupted!"						)
  																									_lowercase					:  Tuple    			=      token_index
  																			writer.write(" ".join(snake_case						) + "\n"						)
  																			index += 1
  
  							return vocab_file, merge_file
  
  
  
  
  
style_context: 
  def 		UpperCamelCase    (					_UpperCAmelCase     :      int     )      -> bool:
  
  
  
  
  
  
  						'''simple docstring'''
  
  						if not isinstance(_UpperCAmelCase   , _UpperCAmelCase     ):
  												_lowercase					:  Optional[int]    			=      f"""Input value of [number={number}] must be an integer"""
  												raise TypeError(_UpperCAmelCase     )
  						if number < 0 or number % 2 == 1:
  												return False
  						_lowercase					:  List[str]    			=      int(number**0.5     )
  						return number == number_sqrt * (number_sqrt + 1)
  
  
  if __name__ == "__main__":
  							import doctest
  
  							doctest.testmod()
  
  
  
  
  