code: class 					UpperCamelCase	:
  
  
  
  
       def __init__( self :					Union[str, Any]						):
  
  
         """simple docstring"""
  
  
  
  
         __snake_case		       =     {}
         # indexmap keys are an index and it's values are the number at that index
         __snake_case		       =     {}
  
  
  
  
       def 							UpperCamelCase_						( self :					Optional[int]      ,_lowerCAmelCase :					Dict      ,_lowerCAmelCase :					Dict						):
  
  
         """simple docstring"""
  
  
  
  
         if isinstance(__snake_case      ,(range, str)						):
           __snake_case		       =     list(__snake_case						)
         elif not isinstance(__snake_case      ,__snake_case						):
           raise TypeError(
               "binary_search_delete() only accepts either a list, range or str"						)
  
         __snake_case		       =     0
         __snake_case		       =     len(__snake_case						) - 1
  
         while low <= high:
           __snake_case		       =     (low + high) // 2
           if array[mid] == item:
             array.pop(__snake_case						)
             return array
           elif array[mid] < item:
             __snake_case		       =     mid + 1
           else:
             __snake_case		       =     mid - 1
         raise ValueError(
             "Either the item is not in the array or the array was unsorted"						)
  
  
  
  
       def 							UpperCamelCase_						( self :					Optional[Any]      ,_lowerCAmelCase :					Union[str, Any]      ,_lowerCAmelCase :					List[str]						):
  
  
         """simple docstring"""
  
  
  
  
         if isinstance(__snake_case      ,(range, str)						):
           __snake_case		       =     list(__snake_case						)
         elif not isinstance(__snake_case      ,__snake_case						):
           raise TypeError(
               "binary_search_insert() only accepts either a list, range or str"						)
  
         __snake_case		       =     0
         __snake_case		       =     len(__snake_case						) - 1
  
         while low <= high:
           __snake_case		       =     (low + high) // 2
           if array[mid] == index:
             # If the item already exists in the array,
             # insert it after the existing item
             array.insert(mid + 1      ,__snake_case						)
             return array
           elif array[mid] < index:
             __snake_case		       =     mid + 1
           else:
             __snake_case		       =     mid - 1
  
          # If the item doesn't exist in the array, insert it at the appropriate position
         array.insert(__snake_case      ,__snake_case						)
         return array
  
  
  
  
       def 							UpperCamelCase_						( self :					str      ,_lowerCAmelCase :					Optional[Any]      ,_lowerCAmelCase :					int						):
  
  
         """simple docstring"""
  
  
  
  
         if index in self.indexmap:
           __snake_case		       =     self.indexmap[index]
           if len(self.numbermap[n]						) == 1:
             del self.numbermap[n]
           else:
             __snake_case		       =     self.binary_search_delete(self.numbermap[n]      ,__snake_case						)
  
          # Set new index
         __snake_case		       =     number
  
         # Number not seen before or empty so insert number value
         if number not in self.numbermap:
           __snake_case		       =     [index]
  
         # Here we need to perform a binary search insertion in order to insert
         # The item in the correct place
         else:
           __snake_case		       =     self.binary_search_insert(
               self.numbermap[number]      ,__snake_case						)
  
  
  
  
  
  
  
       def 							UpperCamelCase_						( self :					int      ,_lowerCAmelCase :					Dict						):
  
  
         """simple docstring"""
  
  
  
  
         return self.numbermap.get(__snake_case      ,[-1]						)[0]
  
  
  if __name__ == "__main__":
    import doctest
  
    doctest.testmod()
  
  
  
  
  
  
style_context: import json
  import os
  import re
  import unicodedata
  from typing import Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
  from ...utils import logging
  
  
  lowerCamelCase__           =				logging.get_logger(__name__)
  
  lowerCamelCase__           =				{
      'vocab_file': 'vocab.json',
      'merges_file': 'merges.txt',
  }
  
  lowerCamelCase__           =				{
      'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
      'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
  }
  
  lowerCamelCase__           =				{
      'openai-gpt': 5_12,
  }
  
  
  
  def 							_lowerCamelCase(				__snake_case	)  ->   Any:
  		__snake_case		       =     text.strip()
  		if not text:
  				return []
  		__snake_case		       =     text.split()
  		return tokens
  
  
  
  
  class 					UpperCamelCase	(     snake_case__      ):
  
  
  
  
  					def __init__( self :					Optional[Any]      ,_lowerCAmelCase :					str=True      ,_lowerCAmelCase :					Any=None      ,_lowerCAmelCase :					Any=True      ,_lowerCAmelCase :					Dict=None      ,_lowerCAmelCase :					List[Any]=True      ,):
  
  
  							"""simple docstring"""
  
  
  
  
  							if never_split is None:
  									__snake_case		       =     []
  							__snake_case		       =     do_lower_case
  							__snake_case		       =     set(_lowerCAmelCase						)
  							__snake_case		       =     tokenize_chinese_chars
  							__snake_case		       =     strip_accents
  							__snake_case		       =     do_split_on_punc
  
  
  
  
  					def 							UpperCamelCase_						( self :					Tuple      ,_lowerCAmelCase :					Union[str, Any]      ,_lowerCAmelCase :					Optional[int]=None						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     self.never_split.union(set(_lowerCAmelCase						)						) if never_split else self.never_split
  							__snake_case		       =     self._clean_text(_lowerCAmelCase						)
  
  							# This was added on November 1st, 2018 for the multilingual and Chinese
  							# models. This is also applied to the English models now, but it doesn't
  							# matter since the English models were not trained on any Chinese data
  							# and generally don't have any Chinese data in them (there are Chinese
  							# characters in the vocabulary because Wikipedia does have some Chinese
  							# words in the English Wikipedia.).
  							if self.tokenize_chinese_chars:
  									__snake_case		       =     self._tokenize_chinese_chars(_lowerCAmelCase						)
  							# prevents treating the same character with different unicode codepoints as different characters
  							__snake_case		       =     unicodedata.normalize("NFC"      ,_lowerCAmelCase						)
  							__snake_case		       =     whitespace_tokenize(_lowerCAmelCase						)
  							__snake_case		       =     []
  							for token in orig_tokens:
  									if token not in never_split:
  											if self.do_lower_case:
  													__snake_case		       =     token.lower()
  													if self.strip_accents is not False:
  															__snake_case		       =     self._run_strip_accents(_lowerCAmelCase						)
  											elif self.strip_accents:
  													__snake_case		       =     self._run_strip_accents(_lowerCAmelCase						)
  									split_tokens.extend(self._run_split_on_punc(_lowerCAmelCase      ,_lowerCAmelCase						)						)
  
  							__snake_case		       =     whitespace_tokenize(" ".join(_lowerCAmelCase						)						)
  							return output_tokens
  
  
  
  
  					def 							UpperCamelCase_						( self :					int      ,_lowerCAmelCase :					Dict						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     unicodedata.normalize("NFD"      ,_lowerCAmelCase						)
  							__snake_case		       =     []
  							for char in text:
  									__snake_case		       =     unicodedata.category(_lowerCAmelCase						)
  									if cat == "Mn":
  											continue
  									output.append(_lowerCAmelCase						)
  							return "".join(_lowerCAmelCase						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					Dict      ,_lowerCAmelCase :					Union[str, Any]      ,_lowerCAmelCase :					List[Any]=None						):
  
  
  							"""simple docstring"""
  
  
  
  
  							if not self.do_split_on_punc or (never_split is not None and text in never_split):
  									return [text]
  							__snake_case		       =     list(_lowerCAmelCase						)
  							__snake_case		       =     0
  							__snake_case		       =     True
  							__snake_case		       =     []
  							while i < len(_lowerCAmelCase						):
  									__snake_case		       =     chars[i]
  									if _is_punctuation(_lowerCAmelCase						):
  											output.append([char]						)
  											__snake_case		       =     True
  									else:
  											if start_new_word:
  													output.append([]						)
  											__snake_case		       =     False
  											output[-1].append(_lowerCAmelCase						)
  									i += 1
  
  							return ["".join(_lowerCAmelCase						) for x in output]
  
  
  
  
  					def 							UpperCamelCase_						( self :					int      ,_lowerCAmelCase :					str						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     []
  							for char in text:
  									__snake_case		       =     ord(_lowerCAmelCase						)
  									if self._is_chinese_char(_lowerCAmelCase						):
  											output.append(" "						)
  											output.append(_lowerCAmelCase						)
  											output.append(" "						)
  									else:
  											output.append(_lowerCAmelCase						)
  							return "".join(_lowerCAmelCase						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					List[str]      ,_lowerCAmelCase :					Optional[int]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							if (
  							    (cp >= 0X4e00 and cp <= 0X9fff)
  							    or (cp >= 0X3400 and cp <= 0X4dbf)  #
  							    or (cp >= 0X2_0000 and cp <= 0X2_a6df)  #
  							    or (cp >= 0X2_a700 and cp <= 0X2_b73f)  #
  							    or (cp >= 0X2_b740 and cp <= 0X2_b81f)  #
  							    or (cp >= 0X2_b820 and cp <= 0X2_ceaf)  #
  							    or (cp >= 0Xf900 and cp <= 0Xfaff)
  							    or (cp >= 0X2_f800 and cp <= 0X2_fa1f)  #
  							):  #
  									return True
  
  							return False
  
  
  
  
  
  
  
  					def 							UpperCamelCase_						( self :					int      ,_lowerCAmelCase :					Optional[int]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     []
  							for char in text:
  									__snake_case		       =     ord(_lowerCAmelCase						)
  									if cp == 0 or cp == 0Xfffd or _is_control(_lowerCAmelCase						):
  											continue
  									if _is_whitespace(_lowerCAmelCase						):
  											output.append(" "						)
  									else:
  											output.append(_lowerCAmelCase						)
  							return "".join(_lowerCAmelCase						)
  
  
  
  
  def 							_lowerCamelCase(				__snake_case	)  ->   str:
  		__snake_case		       =     set()
  		__snake_case		       =     word[0]
  		for char in word[1:]:
  				pairs.add((prev_char, char)	)
  				__snake_case		       =     char
  		return pairs
  
  
  
  def 							_lowerCamelCase(				__snake_case	)  ->   int:
  		__snake_case		       =     text.replace("—"						,						"-"	)
  		__snake_case		       =     text.replace("–"						,						"-"	)
  		__snake_case		       =     text.replace("―"						,						"-"	)
  		__snake_case		       =     text.replace("…"						,						"..."	)
  		__snake_case		       =     text.replace("´"						,						"'"	)
  		__snake_case		       =     re.sub(r"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"						,						r" \1 "						,						__snake_case	)
  		__snake_case		       =     re.sub(r"\s*\n\s*"						,						" \n "						,						__snake_case	)
  		__snake_case		       =     re.sub(r"[^\S\n]+"						,						" "						,						__snake_case	)
  		return text.strip()
  
  
  
  
  
  class 					UpperCamelCase	(     snake_case__      ):
  					__UpperCamelCase						=   VOCAB_FILES_NAMES
  					__UpperCamelCase						=   PRETRAINED_VOCAB_FILES_MAP
  					__UpperCamelCase						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  					__UpperCamelCase						=   ["""input_ids""", """attention_mask"""]
  
  
  
  
  					def __init__( self :					Optional[Any]      ,_lowerCAmelCase :					Union[str, Any]      ,_lowerCAmelCase :					Tuple      ,_lowerCAmelCase :					Tuple="<unk>"      ,**_lowerCAmelCase :					List[str]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							super().__init__(unk_token=_lowerCAmelCase      ,**_lowerCAmelCase						)
  
  							try:
  									import ftfy
  									from spacy.lang.en import English
  
  									__snake_case		       =     English()
  									__snake_case		       =     _nlp.tokenizer
  									__snake_case		       =     ftfy.fix_text
  							except ImportError:
  									logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."						)
  									__snake_case		       =     BasicTokenizer(do_lower_case=_lowerCAmelCase						)
  									__snake_case		       =     None
  
  							with open(_lowerCAmelCase      ,encoding="utf-8"						) as vocab_handle:
  									__snake_case		       =     json.load(_lowerCAmelCase						)
  							__snake_case		       =     {v: k for k, v in self.encoder.items()}
  							with open(_lowerCAmelCase      ,encoding="utf-8"						) as merges_handle:
  									__snake_case		       =     merges_handle.read().split("\n"						)[1:-1]
  							__snake_case		       =     [tuple(merge.split()						) for merge in merges]
  							__snake_case		       =     dict(zip(_lowerCAmelCase      ,range(len(_lowerCAmelCase						)						)						)						)
  							__snake_case		       =     {}
  
  
  
  
  					@property
  					def 							UpperCamelCase_						( self :					Any						):
  
  
  							"""simple docstring"""
  
  
  
  
  							return True
  
  
  
  
  					@property
  					def 							UpperCamelCase_						( self :					Optional[Any]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							return len(self.encoder						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					Any						):
  
  
  							"""simple docstring"""
  
  
  
  
  							return dict(self.encoder      ,**self.added_tokens_encoder						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					Dict      ,_lowerCAmelCase :					Union[str, Any]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     tuple(token[:-1]						) + (token[-1] + "</w>",)
  							if token in self.cache:
  									return self.cache[token]
  							__snake_case		       =     get_pairs(_lowerCAmelCase						)
  
  							if not pairs:
  									return token + "</w>"
  
  							while True:
  									__snake_case		       =     min(_lowerCAmelCase      ,key=lambda _lowerCAmelCase						: self.bpe_ranks.get(_lowerCAmelCase      ,float("inf"						)						)						)
  									if bigram not in self.bpe_ranks:
  											break
  									__snake_case       ,       __snake_case		       =     bigram
  									__snake_case		       =     []
  									__snake_case		       =     0
  									while i < len(_lowerCAmelCase						):
  											try:
  													__snake_case		       =     word.index(_lowerCAmelCase      ,_lowerCAmelCase						)
  											except ValueError:
  													new_word.extend(word[i:]						)
  													break
  											else:
  													new_word.extend(word[i:j]						)
  													__snake_case		       =     j
  
  											if word[i] == first and i < len(_lowerCAmelCase						) - 1 and word[i + 1] == second:
  													new_word.append(first + second						)
  													i += 2
  											else:
  													new_word.append(word[i]						)
  													i += 1
  									__snake_case		       =     tuple(_lowerCAmelCase						)
  									__snake_case		       =     new_word
  									if len(_lowerCAmelCase						) == 1:
  											break
  									else:
  											__snake_case		       =     get_pairs(_lowerCAmelCase						)
  							__snake_case		       =     " ".join(_lowerCAmelCase						)
  							if word == "\n  </w>":
  									__snake_case		       =     "\n</w>"
  							__snake_case		       =     word
  							return word
  
  
  
  
  					def 							UpperCamelCase_						( self :					Union[str, Any]      ,_lowerCAmelCase :					List[Any]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     []
  							if self.fix_text is None:
  									# Using BERT's BasicTokenizer
  									__snake_case		       =     self.nlp.tokenize(_lowerCAmelCase						)
  									for token in text:
  											split_tokens.extend(list(self.bpe(_lowerCAmelCase						).split(" "						)						)						)
  							else:
  									# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
  									__snake_case		       =     self.nlp(text_standardize(self.fix_text(_lowerCAmelCase						)						)						)
  									for token in text:
  											split_tokens.extend(list(self.bpe(token.text.lower()						).split(" "						)						)						)
  							return split_tokens
  
  
  
  
  					def 							UpperCamelCase_						( self :					Dict      ,_lowerCAmelCase :					List[Any]						):
  
  
  							"""simple docstring"""
  
  
  
  
  							return self.encoder.get(_lowerCAmelCase      ,self.encoder.get(self.unk_token						)						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					Dict      ,_lowerCAmelCase :					Any						):
  
  
  							"""simple docstring"""
  
  
  
  
  							return self.decoder.get(_lowerCAmelCase      ,self.unk_token						)
  
  
  
  
  					def 							UpperCamelCase_						( self :					Optional[int]      ,_lowerCAmelCase :					Any						):
  
  
  							"""simple docstring"""
  
  
  
  
  							__snake_case		       =     "".join(_lowerCAmelCase						).replace("</w>"      ," "						).strip()
  							return out_string
  
  
  
  
  
  
  
  					def 							UpperCamelCase_						( self :					Optional[Any]      ,_lowerCAmelCase :					str      ,_lowerCAmelCase :					Optional[str] = None						):
  
  
  							"""simple docstring"""
  
  
  
  
  							if not os.path.isdir(_lowerCAmelCase						):
  									logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""						)
  									return
  							__snake_case		       =     os.path.join(
  							    _lowerCAmelCase      ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
  							__snake_case		       =     os.path.join(
  							    _lowerCAmelCase      ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]						)
  
  							with open(_lowerCAmelCase      ,"w"      ,encoding="utf-8"						) as f:
  									f.write(json.dumps(self.encoder      ,indent=2      ,sort_keys=_lowerCAmelCase      ,ensure_ascii=_lowerCAmelCase						) + "\n"						)
  
  							__snake_case		       =     0
  							with open(_lowerCAmelCase      ,"w"      ,encoding="utf-8"						) as writer:
  									writer.write("#version: 0.2\n"						)
  									for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,key=lambda _lowerCAmelCase						: kv[1]						):
  											if index != token_index:
  													logger.warning(
  													    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
  													    " Please check that the tokenizer is not corrupted!"						)
  													__snake_case		       =     token_index
  											writer.write(" ".join(_lowerCAmelCase						) + "\n"						)
  											index += 1
  
  							return vocab_file, merge_file
  
  
  
  
  
  