code: 
  
  
  
  
  
  '''simple docstring'''
  
  
  import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
  
  
  _lowerCamelCase				:	Any				    =       logging.get_logger(__name__)
  
  _lowerCamelCase				:	List[str]				    =       {
      'vocab_file': 'vocab.json',
      'merges_file': 'merges.txt',
  }
  
  _lowerCamelCase				:	Union[str, Any]				    =       {
      'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
      'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
  }
  
  _lowerCamelCase				:	List[str]				    =       {
      'openai-gpt': 512,
  }
  
  
  
  
  def    __a (     UpperCAmelCase    )      ->Union[str, Any]:
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  
  
  	A							  =      text.strip()
  	if not text:
  		return []
  	A							  =      text.split()
  	return tokens
  
  
  
  class 							__UpperCAmelCase  (       A__						):
  
  
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		def __init__(self						:      Tuple      ,      _lowerCAmelCase						:      Optional[int]=True      ,      _lowerCAmelCase						:      Tuple=None      ,      _lowerCAmelCase						:      str=True      ,      _lowerCAmelCase						:      Tuple=None      ,      _lowerCAmelCase						:      str=True      ,      ):
  			if never_split is None:
  				A							  =      []
  			A							  =      do_lower_case
  			A							  =      set(_lowerCAmelCase      )
  			A							  =      tokenize_chinese_chars
  			A							  =      strip_accents
  			A							  =      do_split_on_punc
  
  
  
  
  
  
  
  		def        A     (self						:      Tuple      ,      _lowerCAmelCase						:      str      ,      _lowerCAmelCase						:      List[Any]=None      ):
  			A							  =      self.never_split.union(set(_lowerCAmelCase      )      ) if never_split else self.never_split
  			A							  =      self._clean_text(_lowerCAmelCase      )
  
  			# This was added on November 1st, 2018 for the multilingual and Chinese
  			# models. This is also applied to the English models now, but it doesn't
  			# matter since the English models were not trained on any Chinese data
  			# and generally don't have any Chinese data in them (there are Chinese
  			# characters in the vocabulary because Wikipedia does have some Chinese
  			# words in the English Wikipedia.).
  			if self.tokenize_chinese_chars:
  				A							  =      self._tokenize_chinese_chars(_lowerCAmelCase      )
  			# prevents treating the same character with different unicode codepoints as different characters
  			A							  =      unicodedata.normalize("""NFC"""      ,      _lowerCAmelCase      )
  			A							  =      whitespace_tokenize(_lowerCAmelCase      )
  			A							  =      []
  			for token in orig_tokens:
  				if token not in never_split:
  					if self.do_lower_case:
  						A							  =      token.lower()
  						if self.strip_accents is not False:
  							A							  =      self._run_strip_accents(_lowerCAmelCase      )
  					elif self.strip_accents:
  						A							  =      self._run_strip_accents(_lowerCAmelCase      )
  				split_tokens.extend(self._run_split_on_punc(_lowerCAmelCase      ,      _lowerCAmelCase      )      )
  
  			A							  =      whitespace_tokenize(""" """.join(_lowerCAmelCase      )      )
  			return output_tokens
  
  
  
  
  
  
  
  		def        A     (self						:      List[Any]      ,      _lowerCAmelCase						:      Union[str, Any]      ):
  			A							  =      unicodedata.normalize("""NFD"""      ,      _lowerCAmelCase      )
  			A							  =      []
  			for char in text:
  				A							  =      unicodedata.category(_lowerCAmelCase      )
  				if cat == "Mn":
  					continue
  				output.append(_lowerCAmelCase      )
  			return "".join(_lowerCAmelCase      )
  
  
  
  
  
  
  
  		def        A     (self						:      Dict      ,      _lowerCAmelCase						:      Dict      ,      _lowerCAmelCase						:      Any=None      ):
  			if not self.do_split_on_punc or (never_split is not None and text in never_split):
  				return [text]
  			A							  =      list(_lowerCAmelCase      )
  			A							  =      0
  			A							  =      True
  			A							  =      []
  			while i < len(_lowerCAmelCase      ):
  				A							  =      chars[i]
  				if _is_punctuation(_lowerCAmelCase      ):
  					output.append([char]      )
  					A							  =      True
  				else:
  					if start_new_word:
  						output.append([]      )
  					A							  =      False
  					output[-1].append(_lowerCAmelCase      )
  				i += 1
  
  			return ["".join(_lowerCAmelCase      ) for x in output]
  
  
  
  
  
  
  
  		def        A     (self						:      List[Any]      ,      _lowerCAmelCase						:      Dict      ):
  			A							  =      []
  			for char in text:
  				A							  =      ord(_lowerCAmelCase      )
  				if self._is_chinese_char(_lowerCAmelCase      ):
  					output.append(""" """      )
  					output.append(_lowerCAmelCase      )
  					output.append(""" """      )
  				else:
  					output.append(_lowerCAmelCase      )
  			return "".join(_lowerCAmelCase      )
  
  
  
  
  
  
  
  		def        A     (self						:      Optional[int]      ,      _lowerCAmelCase						:      Any      ):
  			# This defines a "chinese character" as anything in the CJK Unicode block:
  			#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
  			#
  			# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
  			# despite its name. The modern Korean Hangul alphabet is a different block,
  			# as is Japanese Hiragana and Katakana. Those alphabets are used to write
  			# space-separated words, so they are not treated specially and handled
  			# like the all of the other languages.
  			if (
  			    (cp >= 0X4_e00 and cp <= 0X9_fff)
  			    or (cp >= 0X3_400 and cp <= 0X4_dbf)  #
  			    or (cp >= 0X20_000 and cp <= 0X2a_6df)  #
  			    or (cp >= 0X2a_700 and cp <= 0X2b_73f)  #
  			    or (cp >= 0X2b_740 and cp <= 0X2b_81f)  #
  			    or (cp >= 0X2b_820 and cp <= 0X2c_eaf)  #
  			    or (cp >= 0Xf_900 and cp <= 0Xf_aff)
  			    or (cp >= 0X2f_800 and cp <= 0X2f_a1f)  #
  			):  #
  				return True
  
  			return False
  
  
  
  
  
  
  
  		def        A     (self						:      Dict      ,      _lowerCAmelCase						:      Optional[Any]      ):
  			A							  =      []
  			for char in text:
  				A							  =      ord(_lowerCAmelCase      )
  				if cp == 0 or cp == 0Xf_ffd or _is_control(_lowerCAmelCase      ):
  					continue
  				if _is_whitespace(_lowerCAmelCase      ):
  					output.append(""" """      )
  				else:
  					output.append(_lowerCAmelCase      )
  			return "".join(_lowerCAmelCase      )
  
  
  
  
  
  
  def    __a (     UpperCAmelCase    )      ->str:
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  
  
  	A							  =      set()
  	A							  =      word[0]
  	for char in word[1:]:
  		pairs.add((prev_char, char)    )
  		A							  =      char
  	return pairs
  
  
  
  
  def    __a (     UpperCAmelCase    )      ->List[str]:
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  
  
  	A							  =      text.replace("""—"""       ,			"""-"""    )
  	A							  =      text.replace("""–"""       ,			"""-"""    )
  	A							  =      text.replace("""―"""       ,			"""-"""    )
  	A							  =      text.replace("""…"""       ,			"""..."""    )
  	A							  =      text.replace("""´"""       ,			"""'"""    )
  	A							  =      re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""       ,			R""" \1 """       ,			UpperCAmelCase    )
  	A							  =      re.sub(R"""\s*\n\s*"""       ,			""" \n """       ,			UpperCAmelCase    )
  	A							  =      re.sub(R"""[^\S\n]+"""       ,			""" """       ,			UpperCAmelCase    )
  	return text.strip()
  
  
  
  class 							__UpperCAmelCase  (       A__						):
  
  
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		__lowerCAmelCase 				=    VOCAB_FILES_NAMES
  		__lowerCAmelCase 				=    PRETRAINED_VOCAB_FILES_MAP
  		__lowerCAmelCase 				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  		__lowerCAmelCase 				=    ['''input_ids''', '''attention_mask''']
  
  
  
  
  
  
  
  		def __init__(self						:      Optional[Any]      ,      _lowerCAmelCase						:      Any      ,      _lowerCAmelCase						:      Tuple      ,      _lowerCAmelCase						:      Tuple="<unk>"      ,      **_lowerCAmelCase						:      str      ):
  			super().__init__(unk_token=_lowerCAmelCase      ,      **_lowerCAmelCase      )
  
  			try:
  				import ftfy
  				from spacy.lang.en import English
  
  				A							  =      English()
  				A							  =      _nlp.tokenizer
  				A							  =      ftfy.fix_text
  			except ImportError:
  				logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""      )
  				A							  =      BasicTokenizer(do_lower_case=_lowerCAmelCase      )
  				A							  =      None
  
  			with open(_lowerCAmelCase      ,      encoding="""utf-8"""      ) as vocab_handle:
  				A							  =      json.load(_lowerCAmelCase      )
  			A							  =      {v: k for k, v in self.encoder.items()}
  			with open(_lowerCAmelCase      ,      encoding="""utf-8"""      ) as merges_handle:
  				A							  =      merges_handle.read().split("""\n"""      )[1:-1]
  			A							  =      [tuple(merge.split()      ) for merge in merges]
  			A							  =      dict(zip(_lowerCAmelCase      ,      range(len(_lowerCAmelCase      )      )      )      )
  			A							  =      {}
  
  
  
  
  
  
  
  		@property
  		def        A     (self						:      Any      ):
  			return True
  
  
  
  
  
  
  
  		@property
  		def        A     (self						:      List[Any]      ):
  			return len(self.encoder      )
  
  
  
  
  
  
  
  		def        A     (self						:      List[Any]      ):
  			return dict(self.encoder      ,      **self.added_tokens_encoder      )
  
  
  
  
  
  
  
  		def        A     (self						:      Optional[int]      ,      _lowerCAmelCase						:      Tuple      ):
  			A							  =      tuple(token[:-1]      ) + (token[-1] + """</w>""",)
  			if token in self.cache:
  				return self.cache[token]
  			A							  =      get_pairs(_lowerCAmelCase      )
  
  			if not pairs:
  				return token + "</w>"
  
  			while True:
  				A							  =      min(_lowerCAmelCase      ,      key=lambda _lowerCAmelCase      : self.bpe_ranks.get(_lowerCAmelCase      ,      float("""inf"""      )      )      )
  				if bigram not in self.bpe_ranks:
  					break
  				A  ,    A							  =      bigram
  				A							  =      []
  				A							  =      0
  				while i < len(_lowerCAmelCase      ):
  					try:
  						A							  =      word.index(_lowerCAmelCase      ,      _lowerCAmelCase      )
  					except ValueError:
  						new_word.extend(word[i:]      )
  						break
  					else:
  						new_word.extend(word[i:j]      )
  						A							  =      j
  
  					if word[i] == first and i < len(_lowerCAmelCase      ) - 1 and word[i + 1] == second:
  						new_word.append(first + second      )
  						i += 2
  					else:
  						new_word.append(word[i]      )
  						i += 1
  				A							  =      tuple(_lowerCAmelCase      )
  				A							  =      new_word
  				if len(_lowerCAmelCase      ) == 1:
  					break
  				else:
  					A							  =      get_pairs(_lowerCAmelCase      )
  			A							  =      """ """.join(_lowerCAmelCase      )
  			if word == "\n  </w>":
  				A							  =      """\n</w>"""
  			A							  =      word
  			return word
  
  
  
  
  
  
  
  		def        A     (self						:      str      ,      _lowerCAmelCase						:      List[Any]      ):
  			A							  =      []
  			if self.fix_text is None:
  				# Using BERT's BasicTokenizer
  				A							  =      self.nlp.tokenize(_lowerCAmelCase      )
  				for token in text:
  					split_tokens.extend(list(self.bpe(_lowerCAmelCase      ).split(""" """      )      )      )
  			else:
  				# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
  				A							  =      self.nlp(text_standardize(self.fix_text(_lowerCAmelCase      )      )      )
  				for token in text:
  					split_tokens.extend(list(self.bpe(token.text.lower()      ).split(""" """      )      )      )
  			return split_tokens
  
  
  
  
  
  
  
  		def        A     (self						:      str      ,      _lowerCAmelCase						:      Optional[int]      ):
  			return self.encoder.get(_lowerCAmelCase      ,      self.encoder.get(self.unk_token      )      )
  
  
  
  
  
  
  
  		def        A     (self						:      Optional[int]      ,      _lowerCAmelCase						:      str      ):
  			return self.decoder.get(_lowerCAmelCase      ,      self.unk_token      )
  
  
  
  
  
  
  
  		def        A     (self						:      List[Any]      ,      _lowerCAmelCase						:      List[str]      ):
  			A							  =      """""".join(_lowerCAmelCase      ).replace("""</w>"""      ,      """ """      ).strip()
  			return out_string
  
  
  
  
  
  
  
  		def        A     (self						:      str      ,      _lowerCAmelCase						:      str      ,      _lowerCAmelCase						:      Optional[str] = None      ):
  			if not os.path.isdir(_lowerCAmelCase      ):
  				logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
  				return
  			A							  =      os.path.join(
  			    _lowerCAmelCase      ,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )
  			A							  =      os.path.join(
  			    _lowerCAmelCase      ,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]      )
  
  			with open(_lowerCAmelCase      ,      """w"""      ,      encoding="""utf-8"""      ) as f:
  				f.write(json.dumps(self.encoder      ,      indent=2      ,      sort_keys=_lowerCAmelCase      ,      ensure_ascii=_lowerCAmelCase      ) + """\n"""      )
  
  			A							  =      0
  			with open(_lowerCAmelCase      ,      """w"""      ,      encoding="""utf-8"""      ) as writer:
  				writer.write("""#version: 0.2\n"""      )
  				for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,      key=lambda _lowerCAmelCase      : kv[1]      ):
  					if index != token_index:
  						logger.warning(
  						    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
  						    """ Please check that the tokenizer is not corrupted!"""      )
  						A							  =      token_index
  					writer.write(""" """.join(_lowerCAmelCase      ) + """\n"""      )
  					index += 1
  
  			return vocab_file, merge_file
  
  
style_context: 
  
  
  
  
  
  '''simple docstring'''
  
  
  from __future__ import annotations
  
  
  
  class 							__UpperCAmelCase  :
  
  
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		def __init__(self						:      Any      ,      _lowerCAmelCase						:      int      ):
  			A							  =      value
  			A							  =      None
  			A							  =      None
  
  
  
  
  
  
  class 							__UpperCAmelCase  :
  
  
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		__lowerCAmelCase 				=    42
  
  
  
  
  
  
  
  		def __init__(self						:      List[str]      ):
  			A							  =      0
  
  
  
  
  
  
  
  		def        A     (self						:      Dict      ,      _lowerCAmelCase						:      Node | None      ,      _lowerCAmelCase						:      int      ):
  			if node is None:
  				return
  
  			if path_sum == self.target:
  				self.paths += 1
  
  			if node.left:
  				self.depth_first_search(node.left      ,      path_sum + node.left.value      )
  			if node.right:
  				self.depth_first_search(node.right      ,      path_sum + node.right.value      )
  
  
  
  
  
  
  
  		def        A     (self						:      Optional[Any]      ,      _lowerCAmelCase						:      Node | None      ,      _lowerCAmelCase						:      int | None = None      ):
  			if node is None:
  				return 0
  			if target is not None:
  				A							  =      target
  
  			self.depth_first_search(_lowerCAmelCase      ,      node.value      )
  			self.path_sum(node.left      )
  			self.path_sum(node.right      )
  
  			return self.paths
  
  
  if __name__ == "__main__":
  				import doctest
  
  				doctest.testmod()
  
  