code: 
  """simple docstring"""
  import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
  
  
  __magic_name__		       =				logging.get_logger(__name__)
  
  __magic_name__		       =				{
      '''vocab_file''': '''vocab.json''',
      '''merges_file''': '''merges.txt''',
  }
  
  __magic_name__		       =				{
      '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
      '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
  }
  
  __magic_name__		       =				{
      '''openai-gpt''': 5_12,
  }
  
  
  
  
  def     __lowerCamelCase  (		UpperCamelCase__      ):
  
  
  	"""simple docstring"""
  
  
  
  
  
  
  	_UpperCAmelCase =						text.strip()
  	if not text:
  		return []
  	_UpperCAmelCase =						text.split()
  	return tokens
  
  
  class    _lowerCAmelCase						(						lowerCamelCase__						):
  
  
  
  							def __init__(      self ,						a_=True ,						a_=None ,						a_=True ,						a_=None ,						a_=True ,						)							->							Dict:
  								if never_split is None:
  									_UpperCAmelCase =						[]
  								_UpperCAmelCase =						do_lower_case
  								_UpperCAmelCase =						set(SCREAMING_SNAKE_CASE_   )
  								_UpperCAmelCase =						tokenize_chinese_chars
  								_UpperCAmelCase =						strip_accents
  								_UpperCAmelCase =						do_split_on_punc
  
  
  
  							def       _a   (      self ,						a_ ,						a_=None   )							->							Optional[Any]:
  								_UpperCAmelCase =						self.never_split.union(set(SCREAMING_SNAKE_CASE_   )   ) if never_split else self.never_split
  								_UpperCAmelCase =						self._clean_text(SCREAMING_SNAKE_CASE_   )
  
  								# This was added on November 1st, 2018 for the multilingual and Chinese
  								# models. This is also applied to the English models now, but it doesn't
  								# matter since the English models were not trained on any Chinese data
  								# and generally don't have any Chinese data in them (there are Chinese
  								# characters in the vocabulary because Wikipedia does have some Chinese
  								# words in the English Wikipedia.).
  								if self.tokenize_chinese_chars:
  									_UpperCAmelCase =						self._tokenize_chinese_chars(SCREAMING_SNAKE_CASE_   )
  								# prevents treating the same character with different unicode codepoints as different characters
  								_UpperCAmelCase =						unicodedata.normalize("NFC" ,						SCREAMING_SNAKE_CASE_   )
  								_UpperCAmelCase =						whitespace_tokenize(SCREAMING_SNAKE_CASE_   )
  								_UpperCAmelCase =						[]
  								for token in orig_tokens:
  									if token not in never_split:
  										if self.do_lower_case:
  											_UpperCAmelCase =						token.lower()
  											if self.strip_accents is not False:
  												_UpperCAmelCase =						self._run_strip_accents(SCREAMING_SNAKE_CASE_   )
  										elif self.strip_accents:
  											_UpperCAmelCase =						self._run_strip_accents(SCREAMING_SNAKE_CASE_   )
  									split_tokens.extend(self._run_split_on_punc(SCREAMING_SNAKE_CASE_ ,						SCREAMING_SNAKE_CASE_   )   )
  
  								_UpperCAmelCase =						whitespace_tokenize(" ".join(SCREAMING_SNAKE_CASE_   )   )
  								return output_tokens
  
  
  
  							def       _a   (      self ,						a_   )							->							List[Any]:
  								_UpperCAmelCase =						unicodedata.normalize("NFD" ,						SCREAMING_SNAKE_CASE_   )
  								_UpperCAmelCase =						[]
  								for char in text:
  									_UpperCAmelCase =						unicodedata.category(SCREAMING_SNAKE_CASE_   )
  									if cat == "Mn":
  										continue
  									output.append(SCREAMING_SNAKE_CASE_   )
  								return "".join(SCREAMING_SNAKE_CASE_   )
  
  
  
  							def       _a   (      self ,						a_ ,						a_=None   )							->							int:
  								if not self.do_split_on_punc or (never_split is not None and text in never_split):
  									return [text]
  								_UpperCAmelCase =						list(SCREAMING_SNAKE_CASE_   )
  								_UpperCAmelCase =						0
  								_UpperCAmelCase =						True
  								_UpperCAmelCase =						[]
  								while i < len(SCREAMING_SNAKE_CASE_   ):
  									_UpperCAmelCase =						chars[i]
  									if _is_punctuation(SCREAMING_SNAKE_CASE_   ):
  										output.append([char]   )
  										_UpperCAmelCase =						True
  									else:
  										if start_new_word:
  											output.append([]   )
  										_UpperCAmelCase =						False
  										output[-1].append(SCREAMING_SNAKE_CASE_   )
  									i += 1
  
  								return ["".join(SCREAMING_SNAKE_CASE_   ) for x in output]
  
  
  
  							def       _a   (      self ,						a_   )							->							str:
  								_UpperCAmelCase =						[]
  								for char in text:
  									_UpperCAmelCase =						ord(SCREAMING_SNAKE_CASE_   )
  									if self._is_chinese_char(SCREAMING_SNAKE_CASE_   ):
  										output.append(" "   )
  										output.append(SCREAMING_SNAKE_CASE_   )
  										output.append(" "   )
  									else:
  										output.append(SCREAMING_SNAKE_CASE_   )
  								return "".join(SCREAMING_SNAKE_CASE_   )
  
  
  
  							def       _a   (      self ,						a_   )							->							Dict:
  								# This defines a "chinese character" as anything in the CJK Unicode block:
  								#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
  								#
  								# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
  								# despite its name. The modern Korean Hangul alphabet is a different block,
  								# as is Japanese Hiragana and Katakana. Those alphabets are used to write
  								# space-separated words, so they are not treated specially and handled
  								# like the all of the other languages.
  								if (
  								    (cp >= 0X4_e_0_0 and cp <= 0X9_f_f_f)
  								    or (cp >= 0X3_4_0_0 and cp <= 0X4_d_b_f)  #
  								    or (cp >= 0X2_0_0_0_0 and cp <= 0X2_a_6_d_f)  #
  								    or (cp >= 0X2_a_7_0_0 and cp <= 0X2_b_7_3_f)  #
  								    or (cp >= 0X2_b_7_4_0 and cp <= 0X2_b_8_1_f)  #
  								    or (cp >= 0X2_b_8_2_0 and cp <= 0X2_c_e_a_f)  #
  								    or (cp >= 0Xf_9_0_0 and cp <= 0Xf_a_f_f)
  								    or (cp >= 0X2_f_8_0_0 and cp <= 0X2_f_a_1_f)  #
  								):  #
  									return True
  
  								return False
  
  
  
  							def       _a   (      self ,						a_   )							->							Optional[int]:
  								_UpperCAmelCase =						[]
  								for char in text:
  									_UpperCAmelCase =						ord(SCREAMING_SNAKE_CASE_   )
  									if cp == 0 or cp == 0Xf_f_f_d or _is_control(SCREAMING_SNAKE_CASE_   ):
  										continue
  									if _is_whitespace(SCREAMING_SNAKE_CASE_   ):
  										output.append(" "   )
  									else:
  										output.append(SCREAMING_SNAKE_CASE_   )
  								return "".join(SCREAMING_SNAKE_CASE_   )
  
  
  
  
  
  def     __lowerCamelCase  (		UpperCamelCase__      ):
  
  
  	"""simple docstring"""
  
  
  
  
  
  
  	_UpperCAmelCase =						set()
  	_UpperCAmelCase =						word[0]
  	for char in word[1:]:
  		pairs.add((prev_char, char)      )
  		_UpperCAmelCase =						char
  	return pairs
  
  
  
  
  def     __lowerCamelCase  (		UpperCamelCase__      ):
  
  
  	"""simple docstring"""
  
  
  
  
  
  
  	_UpperCAmelCase =						text.replace("—"				,     "-"      )
  	_UpperCAmelCase =						text.replace("–"				,     "-"      )
  	_UpperCAmelCase =						text.replace("―"				,     "-"      )
  	_UpperCAmelCase =						text.replace("…"				,     "..."      )
  	_UpperCAmelCase =						text.replace("´"				,     "\'"      )
  	_UpperCAmelCase =						re.sub(r"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"				,     r" \1 "				,     _UpperCAmelCase      )
  	_UpperCAmelCase =						re.sub(r"\s*\n\s*"				,     " \n "				,     _UpperCAmelCase      )
  	_UpperCAmelCase =						re.sub(r"[^\S\n]+"				,     " "				,     _UpperCAmelCase      )
  	return text.strip()
  
  
  class    _lowerCAmelCase						(						lowerCamelCase__						):
  							lowercase_     :     Optional[int]       =							VOCAB_FILES_NAMES
  							lowercase_     :     List[str]       =							PRETRAINED_VOCAB_FILES_MAP
  							lowercase_     :     int       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  							lowercase_     :     Any       =							['''input_ids''', '''attention_mask''']
  
  
  
  							def __init__(      self ,						a_ ,						a_ ,						a_="<unk>" ,						**a_   )							->							Tuple:
  								super().__init__(unk_token=SCREAMING_SNAKE_CASE_ ,						**SCREAMING_SNAKE_CASE_   )
  
  								try:
  									import ftfy
  									from spacy.lang.en import English
  
  									_UpperCAmelCase =						English()
  									_UpperCAmelCase =						_nlp.tokenizer
  									_UpperCAmelCase =						ftfy.fix_text
  								except ImportError:
  									logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."   )
  									_UpperCAmelCase =						BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_   )
  									_UpperCAmelCase =						None
  
  								with open(SCREAMING_SNAKE_CASE_ ,						encoding="utf-8"   ) as vocab_handle:
  									_UpperCAmelCase =						json.load(SCREAMING_SNAKE_CASE_   )
  								_UpperCAmelCase =						{v: k for k, v in self.encoder.items()}
  								with open(SCREAMING_SNAKE_CASE_ ,						encoding="utf-8"   ) as merges_handle:
  									_UpperCAmelCase =						merges_handle.read().split("\n"   )[1:-1]
  								_UpperCAmelCase =						[tuple(merge.split()   ) for merge in merges]
  								_UpperCAmelCase =						dict(zip(SCREAMING_SNAKE_CASE_ ,						range(len(SCREAMING_SNAKE_CASE_   )   )   )   )
  								_UpperCAmelCase =						{}
  
  
  
  							@property
  							def       _a   (      self   )							->							str:
  								return True
  
  
  
  							@property
  							def       _a   (      self   )							->							Union[str, Any]:
  								return len(self.encoder   )
  
  
  
  							def       _a   (      self   )							->							Dict:
  								return dict(self.encoder ,						**self.added_tokens_encoder   )
  
  
  
  							def       _a   (      self ,						a_   )							->							List[Any]:
  								_UpperCAmelCase =						tuple(token[:-1]   ) + (token[-1] + "</w>",)
  								if token in self.cache:
  									return self.cache[token]
  								_UpperCAmelCase =						get_pairs(SCREAMING_SNAKE_CASE_   )
  
  								if not pairs:
  									return token + "</w>"
  
  								while True:
  									_UpperCAmelCase =						min(SCREAMING_SNAKE_CASE_ ,						key=lambda a_   : self.bpe_ranks.get(SCREAMING_SNAKE_CASE_ ,						float("inf"   )   )   )
  									if bigram not in self.bpe_ranks:
  										break
  									_UpperCAmelCase ,      _UpperCAmelCase =						bigram
  									_UpperCAmelCase =						[]
  									_UpperCAmelCase =						0
  									while i < len(SCREAMING_SNAKE_CASE_   ):
  										try:
  											_UpperCAmelCase =						word.index(SCREAMING_SNAKE_CASE_ ,						SCREAMING_SNAKE_CASE_   )
  										except ValueError:
  											new_word.extend(word[i:]   )
  											break
  										else:
  											new_word.extend(word[i:j]   )
  											_UpperCAmelCase =						j
  
  										if word[i] == first and i < len(SCREAMING_SNAKE_CASE_   ) - 1 and word[i + 1] == second:
  											new_word.append(first + second   )
  											i += 2
  										else:
  											new_word.append(word[i]   )
  											i += 1
  									_UpperCAmelCase =						tuple(SCREAMING_SNAKE_CASE_   )
  									_UpperCAmelCase =						new_word
  									if len(SCREAMING_SNAKE_CASE_   ) == 1:
  										break
  									else:
  										_UpperCAmelCase =						get_pairs(SCREAMING_SNAKE_CASE_   )
  								_UpperCAmelCase =						" ".join(SCREAMING_SNAKE_CASE_   )
  								if word == "\n  </w>":
  									_UpperCAmelCase =						"\n</w>"
  								_UpperCAmelCase =						word
  								return word
  
  
  
  							def       _a   (      self ,						a_   )							->							Optional[int]:
  								_UpperCAmelCase =						[]
  								if self.fix_text is None:
  									# Using BERT's BasicTokenizer
  									_UpperCAmelCase =						self.nlp.tokenize(SCREAMING_SNAKE_CASE_   )
  									for token in text:
  										split_tokens.extend(list(self.bpe(SCREAMING_SNAKE_CASE_   ).split(" "   )   )   )
  								else:
  									# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
  									_UpperCAmelCase =						self.nlp(text_standardize(self.fix_text(SCREAMING_SNAKE_CASE_   )   )   )
  									for token in text:
  										split_tokens.extend(list(self.bpe(token.text.lower()   ).split(" "   )   )   )
  								return split_tokens
  
  
  
  							def       _a   (      self ,						a_   )							->							Any:
  								return self.encoder.get(SCREAMING_SNAKE_CASE_ ,						self.encoder.get(self.unk_token   )   )
  
  
  
  							def       _a   (      self ,						a_   )							->							Optional[Any]:
  								return self.decoder.get(SCREAMING_SNAKE_CASE_ ,						self.unk_token   )
  
  
  
  							def       _a   (      self ,						a_   )							->							Union[str, Any]:
  								_UpperCAmelCase =						"".join(SCREAMING_SNAKE_CASE_   ).replace("</w>" ,						" "   ).strip()
  								return out_string
  
  
  
  							def       _a   (      self ,						a_ ,						a_ = None   )							->							Any:
  								if not os.path.isdir(SCREAMING_SNAKE_CASE_   ):
  									logger.error(f"Vocabulary path ({save_directory}) should be a directory"   )
  									return
  								_UpperCAmelCase =						os.path.join(
  								    SCREAMING_SNAKE_CASE_ ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )
  								_UpperCAmelCase =						os.path.join(
  								    SCREAMING_SNAKE_CASE_ ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]   )
  
  								with open(SCREAMING_SNAKE_CASE_ ,						"w" ,						encoding="utf-8"   ) as f:
  									f.write(json.dumps(self.encoder ,						indent=2 ,						sort_keys=SCREAMING_SNAKE_CASE_ ,						ensure_ascii=SCREAMING_SNAKE_CASE_   ) + "\n"   )
  
  								_UpperCAmelCase =						0
  								with open(SCREAMING_SNAKE_CASE_ ,						"w" ,						encoding="utf-8"   ) as writer:
  									writer.write("#version: 0.2\n"   )
  									for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,						key=lambda a_   : kv[1]   ):
  										if index != token_index:
  											logger.warning(
  											    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
  											    " Please check that the tokenizer is not corrupted!"   )
  											_UpperCAmelCase =						token_index
  										writer.write(" ".join(SCREAMING_SNAKE_CASE_   ) + "\n"   )
  										index += 1
  
  								return vocab_file, merge_file
  
  
  
  
  
style_context: 
  """simple docstring"""
  from .stack import Stack
  
  
  
  
  def     __lowerCamelCase  (		UpperCamelCase__      ):
  
  
  	"""simple docstring"""
  
  
  
  
  
  
  	_UpperCAmelCase =						Stack()
  	_UpperCAmelCase =						{"(": ")", "[": "]", "{": "}"}
  	for bracket in parentheses:
  		if bracket in bracket_pairs:
  			stack.push(UpperCamelCase__      )
  		elif bracket in (")", "]", "}"):
  			if stack.is_empty() or bracket_pairs[stack.pop()] != bracket:
  				return False
  	return stack.is_empty()
  
  
  if __name__ == "__main__":
  			from doctest import testmod
  
  			testmod()
  
  			__magic_name__		       =				['''((()))''', '''((())''', '''(()))''']
  			print('''Balanced parentheses demonstration:\n''')
  			for example in examples:
  						__magic_name__		       =				'''''' if balanced_parentheses(example) else '''not '''
  						print(f'''{example} is {not_str}balanced''')
  
  
  
  
  