code: 
  import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
  
  
  _snake_case			:       List[str]      =						logging.get_logger(__name__)
  
  _snake_case			:       int      =						{
      "vocab_file": "vocab.json",
      "merges_file": "merges.txt",
  }
  
  _snake_case			:       List[Any]      =						{
      "vocab_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/vocab.json"},
      "merges_file": {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/merges.txt"},
  }
  
  _snake_case			:       Union[str, Any]      =						{
      "openai-gpt": 5_12,
  }
  
  def    __snake_case       (  __magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				lowercase						    =			text.strip()
  				if not text:
  								return []
  				lowercase						    =			text.split()
  				return tokens
  
  
  
  
  
  class      UpperCamelCase_			(			__a		):
  
  
  
  
  
  
  
  						'''simple docstring'''
  
  
  
  						def __init__( self       :str		,						lowerCAmelCase__       :Optional[Any]=True		,						lowerCAmelCase__       :Optional[int]=None		,						lowerCAmelCase__       :Any=True		,						lowerCAmelCase__       :Union[str, Any]=None		,						lowerCAmelCase__       :int=True		,						)     ->Union[str, Any]:
  										if never_split is None:
  														lowercase						    =			[]
  										lowercase						    =			do_lower_case
  										lowercase						    =			set(lowerCAmelCase__    )
  										lowercase						    =			tokenize_chinese_chars
  										lowercase						    =			strip_accents
  										lowercase						    =			do_split_on_punc
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :str		,						lowerCAmelCase__       :Optional[Any]		,						lowerCAmelCase__       :List[str]=None    )     ->Optional[int]:
  										lowercase						    =			self.never_split.union(set(lowerCAmelCase__    )    ) if never_split else self.never_split
  										lowercase						    =			self._clean_text(lowerCAmelCase__    )
  
  										# This was added on November 1st, 2018 for the multilingual and Chinese
  										# models. This is also applied to the English models now, but it doesn't
  										# matter since the English models were not trained on any Chinese data
  										# and generally don't have any Chinese data in them (there are Chinese
  										# characters in the vocabulary because Wikipedia does have some Chinese
  										# words in the English Wikipedia.).
  										if self.tokenize_chinese_chars:
  														lowercase						    =			self._tokenize_chinese_chars(lowerCAmelCase__    )
  										# prevents treating the same character with different unicode codepoints as different characters
  										lowercase						    =			unicodedata.normalize("NFC"		,						lowerCAmelCase__    )
  										lowercase						    =			whitespace_tokenize(lowerCAmelCase__    )
  										lowercase						    =			[]
  										for token in orig_tokens:
  														if token not in never_split:
  																		if self.do_lower_case:
  																						lowercase						    =			token.lower()
  																						if self.strip_accents is not False:
  																										lowercase						    =			self._run_strip_accents(lowerCAmelCase__    )
  																		elif self.strip_accents:
  																						lowercase						    =			self._run_strip_accents(lowerCAmelCase__    )
  														split_tokens.extend(self._run_split_on_punc(lowerCAmelCase__		,						lowerCAmelCase__    )    )
  
  										lowercase						    =			whitespace_tokenize(" ".join(lowerCAmelCase__    )    )
  										return output_tokens
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Union[str, Any]		,						lowerCAmelCase__       :Any    )     ->Tuple:
  										lowercase						    =			unicodedata.normalize("NFD"		,						lowerCAmelCase__    )
  										lowercase						    =			[]
  										for char in text:
  														lowercase						    =			unicodedata.category(lowerCAmelCase__    )
  														if cat == "Mn":
  																		continue
  														output.append(lowerCAmelCase__    )
  										return "".join(lowerCAmelCase__    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Tuple		,						lowerCAmelCase__       :str		,						lowerCAmelCase__       :Any=None    )     ->Tuple:
  										if not self.do_split_on_punc or (never_split is not None and text in never_split):
  														return [text]
  										lowercase						    =			list(lowerCAmelCase__    )
  										lowercase						    =			0
  										lowercase						    =			True
  										lowercase						    =			[]
  										while i < len(lowerCAmelCase__    ):
  														lowercase						    =			chars[i]
  														if _is_punctuation(lowerCAmelCase__    ):
  																		output.append([char]    )
  																		lowercase						    =			True
  														else:
  																		if start_new_word:
  																						output.append([]    )
  																		lowercase						    =			False
  																		output[-1].append(lowerCAmelCase__    )
  														i += 1
  
  										return ["".join(lowerCAmelCase__    ) for x in output]
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Dict		,						lowerCAmelCase__       :List[str]    )     ->Union[str, Any]:
  										lowercase						    =			[]
  										for char in text:
  														lowercase						    =			ord(lowerCAmelCase__    )
  														if self._is_chinese_char(lowerCAmelCase__    ):
  																		output.append(" "    )
  																		output.append(lowerCAmelCase__    )
  																		output.append(" "    )
  														else:
  																		output.append(lowerCAmelCase__    )
  										return "".join(lowerCAmelCase__    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :str		,						lowerCAmelCase__       :Optional[int]    )     ->List[Any]:
  										# This defines a "chinese character" as anything in the CJK Unicode block:
  										#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
  										#
  										# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
  										# despite its name. The modern Korean Hangul alphabet is a different block,
  										# as is Japanese Hiragana and Katakana. Those alphabets are used to write
  										# space-separated words, so they are not treated specially and handled
  										# like the all of the other languages.
  										if (
  										    (cp >= 0x4E00 and cp <= 0x9FFF)
  										    or (cp >= 0x3400 and cp <= 0x4DBF)  #
  										    or (cp >= 0x2_0000 and cp <= 0x2_A6DF)  #
  										    or (cp >= 0x2_A700 and cp <= 0x2_B73F)  #
  										    or (cp >= 0x2_B740 and cp <= 0x2_B81F)  #
  										    or (cp >= 0x2_B820 and cp <= 0x2_CEAF)  #
  										    or (cp >= 0xF900 and cp <= 0xFAFF)
  										    or (cp >= 0x2_F800 and cp <= 0x2_FA1F)  #
  										):  #
  														return True
  
  										return False
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :List[Any]		,						lowerCAmelCase__       :List[Any]    )     ->List[str]:
  										lowercase						    =			[]
  										for char in text:
  														lowercase						    =			ord(lowerCAmelCase__    )
  														if cp == 0 or cp == 0xFFFD or _is_control(lowerCAmelCase__    ):
  																		continue
  														if _is_whitespace(lowerCAmelCase__    ):
  																		output.append(" "    )
  														else:
  																		output.append(lowerCAmelCase__    )
  										return "".join(lowerCAmelCase__    )
  
  
  
  
  def    __snake_case       (  __magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				lowercase						    =			set()
  				lowercase						    =			word[0]
  				for char in word[1:]:
  								pairs.add((prev_char, char)	)
  								lowercase						    =			char
  				return pairs
  
  def    __snake_case       (  __magic_name__	):
  
  
  
  
  
  
  
  				'''simple docstring'''
  
  
  				lowercase						    =			text.replace("—"		,						"-"	)
  				lowercase						    =			text.replace("–"		,						"-"	)
  				lowercase						    =			text.replace("―"		,						"-"	)
  				lowercase						    =			text.replace("…"		,						"..."	)
  				lowercase						    =			text.replace("´"		,						"'"	)
  				lowercase						    =			re.sub(r"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"		,						r" \1 "		,						__magic_name__	)
  				lowercase						    =			re.sub(r"\s*\n\s*"		,						" \n "		,						__magic_name__	)
  				lowercase						    =			re.sub(r"[^\S\n]+"		,						" "		,						__magic_name__	)
  				return text.strip()
  
  
  
  
  
  class      UpperCamelCase_			(			__a		):
  
  
  
  
  
  
  
  						'''simple docstring'''
  
  
  
  						UpperCamelCase			:     List[Any]							      =				VOCAB_FILES_NAMES
  						UpperCamelCase			:     Union[str, Any]							      =				PRETRAINED_VOCAB_FILES_MAP
  						UpperCamelCase			:     List[Any]							      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  						UpperCamelCase			:     int							      =				['''input_ids''', '''attention_mask''']
  
  
  
  
  
  						def __init__( self       :Optional[Any]		,						lowerCAmelCase__       :Tuple		,						lowerCAmelCase__       :Dict		,						lowerCAmelCase__       :List[str]="<unk>"		,						**lowerCAmelCase__       :Optional[Any]    )     ->Tuple:
  										super().__init__(unk_token=lowerCAmelCase__		,						**lowerCAmelCase__    )
  
  										try:
  														import ftfy
  														from spacy.lang.en import English
  
  														lowercase						    =			English()
  														lowercase						    =			_nlp.tokenizer
  														lowercase						    =			ftfy.fix_text
  										except ImportError:
  														logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."    )
  														lowercase						    =			BasicTokenizer(do_lower_case=lowerCAmelCase__    )
  														lowercase						    =			None
  
  										with open(lowerCAmelCase__		,						encoding="utf-8"    ) as vocab_handle:
  														lowercase						    =			json.load(lowerCAmelCase__    )
  										lowercase						    =			{v: k for k, v in self.encoder.items()}
  										with open(lowerCAmelCase__		,						encoding="utf-8"    ) as merges_handle:
  														lowercase						    =			merges_handle.read().split("\n"    )[1:-1]
  										lowercase						    =			[tuple(merge.split()    ) for merge in merges]
  										lowercase						    =			dict(zip(lowerCAmelCase__		,						range(len(lowerCAmelCase__    )    )    )    )
  										lowercase						    =			{}
  
  
  
  
  
  						@property
  						def 					SCREAMING_SNAKE_CASE( self       :List[str]    )     ->Optional[Any]:
  										return True
  
  
  
  
  
  						@property
  						def 					SCREAMING_SNAKE_CASE( self       :Dict    )     ->Optional[int]:
  										return len(self.encoder    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :List[str]    )     ->Dict:
  										return dict(self.encoder		,						**self.added_tokens_encoder    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Optional[Any]		,						lowerCAmelCase__       :Union[str, Any]    )     ->Optional[Any]:
  										lowercase						    =			tuple(token[:-1]    ) + (token[-1] + "</w>",)
  										if token in self.cache:
  														return self.cache[token]
  										lowercase						    =			get_pairs(lowerCAmelCase__    )
  
  										if not pairs:
  														return token + "</w>"
  
  										while True:
  														lowercase						    =			min(lowerCAmelCase__		,						key=lambda lowerCAmelCase__    : self.bpe_ranks.get(lowerCAmelCase__		,						float("inf"    )    )    )
  														if bigram not in self.bpe_ranks:
  																		break
  														lowercase    ,  lowercase						    =			bigram
  														lowercase						    =			[]
  														lowercase						    =			0
  														while i < len(lowerCAmelCase__    ):
  																		try:
  																						lowercase						    =			word.index(lowerCAmelCase__		,						lowerCAmelCase__    )
  																		except ValueError:
  																						new_word.extend(word[i:]    )
  																						break
  																		else:
  																						new_word.extend(word[i:j]    )
  																						lowercase						    =			j
  
  																		if word[i] == first and i < len(lowerCAmelCase__    ) - 1 and word[i + 1] == second:
  																						new_word.append(first + second    )
  																						i += 2
  																		else:
  																						new_word.append(word[i]    )
  																						i += 1
  														lowercase						    =			tuple(lowerCAmelCase__    )
  														lowercase						    =			new_word
  														if len(lowerCAmelCase__    ) == 1:
  																		break
  														else:
  																		lowercase						    =			get_pairs(lowerCAmelCase__    )
  										lowercase						    =			" ".join(lowerCAmelCase__    )
  										if word == "\n  </w>":
  														lowercase						    =			"\n</w>"
  										lowercase						    =			word
  										return word
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :List[Any]		,						lowerCAmelCase__       :str    )     ->List[Any]:
  										lowercase						    =			[]
  										if self.fix_text is None:
  														# Using BERT's BasicTokenizer
  														lowercase						    =			self.nlp.tokenize(lowerCAmelCase__    )
  														for token in text:
  																		split_tokens.extend(list(self.bpe(lowerCAmelCase__    ).split(" "    )    )    )
  										else:
  														# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
  														lowercase						    =			self.nlp(text_standardize(self.fix_text(lowerCAmelCase__    )    )    )
  														for token in text:
  																		split_tokens.extend(list(self.bpe(token.text.lower()    ).split(" "    )    )    )
  										return split_tokens
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Tuple		,						lowerCAmelCase__       :List[Any]    )     ->Union[str, Any]:
  										return self.encoder.get(lowerCAmelCase__		,						self.encoder.get(self.unk_token    )    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Dict		,						lowerCAmelCase__       :Any    )     ->Dict:
  										return self.decoder.get(lowerCAmelCase__		,						self.unk_token    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :int		,						lowerCAmelCase__       :List[str]    )     ->Optional[int]:
  										lowercase						    =			"".join(lowerCAmelCase__    ).replace("</w>"		,						" "    ).strip()
  										return out_string
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Union[str, Any]		,						lowerCAmelCase__       :str		,						lowerCAmelCase__       :Optional[str] = None    )     ->Tuple[str]:
  										if not os.path.isdir(lowerCAmelCase__    ):
  														logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
  														return
  										lowercase						    =			os.path.join(
  										    lowerCAmelCase__		,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )
  										lowercase						    =			os.path.join(
  										    lowerCAmelCase__		,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]    )
  
  										with open(lowerCAmelCase__		,						"w"		,						encoding="utf-8"    ) as f:
  														f.write(json.dumps(self.encoder		,						indent=2		,						sort_keys=lowerCAmelCase__		,						ensure_ascii=lowerCAmelCase__    ) + "\n"    )
  
  										lowercase						    =			0
  										with open(lowerCAmelCase__		,						"w"		,						encoding="utf-8"    ) as writer:
  														writer.write("#version: 0.2\n"    )
  														for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,						key=lambda lowerCAmelCase__    : kv[1]    ):
  																		if index != token_index:
  																						logger.warning(
  																						    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
  																						    " Please check that the tokenizer is not corrupted!"    )
  																						lowercase						    =			token_index
  																		writer.write(" ".join(lowerCAmelCase__    ) + "\n"    )
  																		index += 1
  
  										return vocab_file, merge_file
  
style_context: 
  import sys
  from collections.abc import Mapping
  
  import numpy as np
  import pyarrow as pa
  
  from .. import config
  from ..utils.py_utils import map_nested
  from .formatting import TensorFormatter
  
  
  
  
  
  class      UpperCamelCase_			(			TensorFormatter[Mapping, np.ndarray, Mapping]		):
  
  
  
  
  
  
  
  						'''simple docstring'''
  
  
  
  						def __init__( self       :int		,						lowerCAmelCase__       :Any=None		,						**lowerCAmelCase__       :Union[str, Any]    )     ->int:
  										super().__init__(features=lowerCAmelCase__    )
  										lowercase						    =			np_array_kwargs
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Union[str, Any]		,						lowerCAmelCase__       :str    )     ->Optional[Any]:
  										if isinstance(lowerCAmelCase__		,						lowerCAmelCase__    ):
  														if column and all(
  														    isinstance(lowerCAmelCase__		,						np.ndarray    ) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column    ):
  																		return np.stack(lowerCAmelCase__    )
  														else:
  																		# don't use np.array(column, dtype=object)
  																		# since it fails in certain cases
  																		# see https://stackoverflow.com/q/51005699
  																		lowercase						    =			np.empty(len(lowerCAmelCase__    )		,						dtype=lowerCAmelCase__    )
  																		lowercase						    =			column
  																		return out
  										return column
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :str		,						lowerCAmelCase__       :List[str]    )     ->Union[str, Any]:
  										if isinstance(lowerCAmelCase__		,						(str, bytes, type(lowerCAmelCase__    ))    ):
  														return value
  										elif isinstance(lowerCAmelCase__		,						(np.character, np.ndarray)    ) and np.issubdtype(value.dtype		,						np.character    ):
  														return value
  										elif isinstance(lowerCAmelCase__		,						np.number    ):
  														return value
  
  										lowercase						    =			{}
  
  										if isinstance(lowerCAmelCase__		,						np.ndarray    ) and np.issubdtype(value.dtype		,						np.integer    ):
  														lowercase						    =			{"dtype": np.intaa}
  										elif isinstance(lowerCAmelCase__		,						np.ndarray    ) and np.issubdtype(value.dtype		,						np.floating    ):
  														lowercase						    =			{"dtype": np.floataa}
  										elif config.PIL_AVAILABLE and "PIL" in sys.modules:
  														import PIL.Image
  
  														if isinstance(lowerCAmelCase__		,						PIL.Image.Image    ):
  																		return np.asarray(lowerCAmelCase__		,						**self.np_array_kwargs    )
  
  										return np.asarray(lowerCAmelCase__		,						**{**default_dtype, **self.np_array_kwargs}    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :int		,						lowerCAmelCase__       :Dict    )     ->int:
  										# support for torch, tf, jax etc.
  										if config.TORCH_AVAILABLE and "torch" in sys.modules:
  														import torch
  
  														if isinstance(lowerCAmelCase__		,						torch.Tensor    ):
  																		return self._tensorize(data_struct.detach().cpu().numpy()[()]    )
  										if hasattr(lowerCAmelCase__		,						"__array__"    ) and not isinstance(lowerCAmelCase__		,						(np.ndarray, np.character, np.number)    ):
  														lowercase						    =			data_struct.__array__()
  										# support for nested types like struct of list of struct
  										if isinstance(lowerCAmelCase__		,						np.ndarray    ):
  														if data_struct.dtype == object:
  																		return self._consolidate([self.recursive_tensorize(lowerCAmelCase__    ) for substruct in data_struct]    )
  										if isinstance(lowerCAmelCase__		,						(list, tuple)    ):
  														return self._consolidate([self.recursive_tensorize(lowerCAmelCase__    ) for substruct in data_struct]    )
  										return self._tensorize(lowerCAmelCase__    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :int		,						lowerCAmelCase__       :dict    )     ->str:
  										return map_nested(self._recursive_tensorize		,						lowerCAmelCase__		,						map_list=lowerCAmelCase__    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :Tuple		,						lowerCAmelCase__       :pa.Table    )     ->Mapping:
  										lowercase						    =			self.numpy_arrow_extractor().extract_row(lowerCAmelCase__    )
  										lowercase						    =			self.python_features_decoder.decode_row(lowerCAmelCase__    )
  										return self.recursive_tensorize(lowerCAmelCase__    )
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :str		,						lowerCAmelCase__       :pa.Table    )     ->np.ndarray:
  										lowercase						    =			self.numpy_arrow_extractor().extract_column(lowerCAmelCase__    )
  										lowercase						    =			self.python_features_decoder.decode_column(lowerCAmelCase__		,						pa_table.column_names[0]    )
  										lowercase						    =			self.recursive_tensorize(lowerCAmelCase__    )
  										lowercase						    =			self._consolidate(lowerCAmelCase__    )
  										return column
  
  
  
  
  
  						def 					SCREAMING_SNAKE_CASE( self       :List[Any]		,						lowerCAmelCase__       :pa.Table    )     ->Mapping:
  										lowercase						    =			self.numpy_arrow_extractor().extract_batch(lowerCAmelCase__    )
  										lowercase						    =			self.python_features_decoder.decode_batch(lowerCAmelCase__    )
  										lowercase						    =			self.recursive_tensorize(lowerCAmelCase__    )
  										for column_name in batch:
  														lowercase						    =			self._consolidate(batch[column_name]    )
  										return batch
  